From fa0caf3b4af9f8172370a2fcac58762abf9fc706 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Mon, 27 Oct 2025 02:43:34 +0300 Subject: [PATCH 01/11] Update format.yml --- .github/workflows/format.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index d1ad75ea..7fe7e7ac 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -31,7 +31,6 @@ jobs: - name: Get list of changed *.channels.xml files id: files run: | - git fetch origin master:master CHANNELS_ANY_CHANGED=false CHANNELS_ALL_CHANGED_FILES=$(git diff --diff-filter=ACMRT --name-only master -- sites/**/*.channels.xml | tr '\n' ' ') if [ -n "${CHANNELS_ALL_CHANGED_FILES}" ]; then From ef1f69ece6281e2e910b53c449fb8f6d34e5ef33 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Tue, 28 Oct 2025 00:32:31 +0300 Subject: [PATCH 02/11] Update grab.ts --- scripts/commands/epg/grab.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/commands/epg/grab.ts b/scripts/commands/epg/grab.ts index 65ad863e..0b90bc19 100644 --- a/scripts/commands/epg/grab.ts +++ b/scripts/commands/epg/grab.ts @@ -133,12 +133,13 @@ async function main() { siteConfig.filepath = channel.getConfigPath() - if (options.timeout !== undefined) { + if (typeof options.timeout === 'number') { siteConfig.request = { ...siteConfig.request, ...{ timeout: options.timeout } } } - if (options.delay !== undefined) siteConfig.delay = options.delay - if (options.curl !== undefined) siteConfig.curl = options.curl - if (options.proxy !== undefined) { + if (typeof options.days === 'number') siteConfig.days = options.days + if (typeof options.delay === 'number') siteConfig.delay = options.delay + if (typeof options.curl === 'boolean') siteConfig.curl = options.curl + if (typeof options.proxy === 'string') { const proxy = parseProxy(options.proxy) if ( @@ -158,9 +159,8 @@ async function main() { if (!channel.xmltv_id) channel.xmltv_id = channel.site_id - const days = options.days || siteConfig.days || 1 const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString()) - const dates = Array.from({ length: days }, (_, day) => currDate.add(day, 'd')) + const dates = Array.from({ length: siteConfig.days }, (_, day) => currDate.add(day, 'd')) dates.forEach((date: Dayjs) => { const key = `${channel.site}:${channel.lang}:${channel.xmltv_id}:${date.toJSON()}` From 947926de8f968a72e328f4070ac1677e61374320 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Tue, 28 Oct 2025 00:32:44 +0300 Subject: [PATCH 03/11] Update Dockerfile --- Dockerfile | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4e91a2e5..74555a83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,22 @@ -FROM node:22-alpine -ARG GIT_REPO=https://github.com/iptv-org/epg.git -ARG GIT_BRANCH=master -ARG WORKDIR=/epg -ENV CRON_SCHEDULE="0 0 * * *" -ENV GZIP=false -ENV MAX_CONNECTIONS=1 -ENV DAYS= -ENV RUN_AT_STARTUP=true -RUN apk update \ - && apk upgrade --available \ - && apk add curl git tzdata bash \ - && npm install -g npm@latest \ - && npm install pm2 -g \ - && mkdir $(echo "${WORKDIR}") -p \ - && cd $WORKDIR \ - && git clone --depth 1 -b $(echo "${GIT_BRANCH} ${GIT_REPO}") . \ - && npm install \ - && mkdir /public -RUN apk del git curl \ - && rm -rf /var/cache/apk/* -COPY pm2.config.js $WORKDIR -WORKDIR $WORKDIR -EXPOSE 3000 +FROM node:22-alpine +ARG GIT_REPO=https://github.com/iptv-org/epg.git +ARG GIT_BRANCH=master +ARG WORKDIR=/epg +ENV CRON_SCHEDULE="0 0 * * *" +ENV RUN_AT_STARTUP=true +RUN apk update \ + && apk upgrade --available \ + && apk add curl git tzdata bash \ + && npm install -g npm@latest \ + && npm install pm2 -g \ + && mkdir $(echo "${WORKDIR}") -p \ + && cd $WORKDIR \ + && git clone --depth 1 -b $(echo "${GIT_BRANCH} ${GIT_REPO}") . \ + && npm install \ + && mkdir /public +RUN apk del git curl \ + && rm -rf /var/cache/apk/* +COPY pm2.config.js $WORKDIR +WORKDIR $WORKDIR +EXPOSE 3000 CMD [ "pm2-runtime", "pm2.config.js" ] \ No newline at end of file From 2f44e5013e13844b5d91e09f7681c0129fc34283 Mon Sep 17 00:00:00 2001 From: "iptv-bot[bot]" <84861620+iptv-bot[bot]@users.noreply.github.com> Date: Tue, 28 Oct 2025 01:01:10 +0000 Subject: [PATCH 04/11] [Bot] Update SITES.md Committed by [iptv-bot](https://github.com/apps/iptv-bot) via [update](https://github.com/iptv-org/epg/actions/runs/18860634783) workflow. --- SITES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SITES.md b/SITES.md index deadf7a6..3800229e 100644 --- a/SITES.md +++ b/SITES.md @@ -207,7 +207,7 @@ tvkaista.org1490🟒 tvmi.mt33🟒 tvmusor.hu9967🟒 - tvmustra.hu1880🟒 + tvmustra.hu1880🟒https://github.com/iptv-org/epg/issues/2903 tvpassport.com192872509🟒https://github.com/iptv-org/epg/issues/2883 tvplus.com.tr150144🟒 tvprofil.com9091408🟒 From 4291cdd32fe1b30c251a8ac9b05796091a08dcc0 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Wed, 29 Oct 2025 07:56:49 +0300 Subject: [PATCH 05/11] Update dependencies --- package-lock.json | 131 ++++++++++++++++++++++++++++++++++++++++------ package.json | 3 +- 2 files changed, 116 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index 51e9bdb7..f19d445b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,9 +48,10 @@ "consola": "^3.4.2", "cross-env": "^10.0.0", "csv-parser": "^3.2.0", + "curl-generator": "^0.5.0", "cwait": "^1.1.2", "dayjs": "^1.11.13", - "epg-grabber": "^0.44.0", + "epg-grabber": "^0.45.0", "epg-parser": "^0.3.1", "eslint": "^9.32.0", "eslint-config-prettier": "^10.1.8", @@ -3425,6 +3426,14 @@ "@types/lodash": "*" } }, + "node_modules/@types/lodash.padstart": { + "version": "4.6.9", + "resolved": "https://registry.npmjs.org/@types/lodash.padstart/-/lodash.padstart-4.6.9.tgz", + "integrity": "sha512-KVXQ65AiorTc+Dn9eSRZDs1SnzXULRJcMYhCDEIgsRtHU7mbVpghPSxkySh3Vgm+doWVzpJCA24259fkRL46sA==", + "dependencies": { + "@types/lodash": "*" + } + }, "node_modules/@types/lodash.sortby": { "version": "4.7.9", "resolved": "https://registry.npmjs.org/@types/lodash.sortby/-/lodash.sortby-4.7.9.tgz", @@ -4243,6 +4252,18 @@ "tough-cookie": ">=4.0.0" } }, + "node_modules/axios-mock-adapter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-2.1.0.tgz", + "integrity": "sha512-AZUe4OjECGCNNssH8SOdtneiQELsqTsat3SQQCWLPjN436/H+L9AjWfV7bF+Zg/YL9cgbhrz5671hoh+Tbn98w==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "is-buffer": "^2.0.5" + }, + "peerDependencies": { + "axios": ">= 0.17.0" + } + }, "node_modules/babel-jest": { "version": "30.0.5", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.0.5.tgz", @@ -5192,9 +5213,9 @@ "integrity": "sha512-yi1x3EAWKjQTreYWeSd98431AV+IEE0qoDyOoaHJ7KJ21gv6HtBXHVLX74opVSGqcR8/AbjJBHAHpcOy2bj5Gg==" }, "node_modules/curl-generator": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/curl-generator/-/curl-generator-0.4.2.tgz", - "integrity": "sha512-YD74vaPyL46XYNbyRCJV91EhYGDrE/EBiW0X/NUIrNZ23jD0Uwr/6vMrCmobYi5KrjtrqN4SnmMhQNYh3qaULw==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/curl-generator/-/curl-generator-0.5.0.tgz", + "integrity": "sha512-dKmp63AJfNSplajvPoWIRfBOmp0IT8NETJ9sFw9IABEUTiJqtXdJx09mXxUlTgVCZDYWVxkCPds+mO9puf1J+w==", "dependencies": { "ms": "^2.0.0" } @@ -5456,17 +5477,19 @@ } }, "node_modules/epg-grabber": { - "version": "0.44.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.44.0.tgz", - "integrity": "sha512-M4k/PG1/OIbHV7p8rM23yDWig6WwtpB/LafRzealc8A50HDOGskmkvuhzxf1S34Oe8xL3aU529oW2vocrpijuA==", + "version": "0.45.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.45.0.tgz", + "integrity": "sha512-GqjXRYOJcC3mX9OYdZHlpNEhwBbCF256uKGPXdUodUjkXTmraolIEXYRUeQJmPbL5/do766EewNwIBXqLzHkJA==", "dependencies": { "@freearhey/core": "^0.14.0", "@types/bluebird": "^3.5.42", "@types/fs-extra": "^11.0.4", - "@types/lodash": "^4.17.20", + "@types/lodash.merge": "^4.6.9", + "@types/lodash.padstart": "^4.6.9", "@types/pako": "^2.0.4", "axios": "^1.12.2", "axios-cache-interceptor": "^1.8.3", + "axios-mock-adapter": "^2.1.0", "bluebird": "^3.7.2", "commander": "^14.0.0", "curl-generator": "^0.4.2", @@ -5474,7 +5497,8 @@ "dayjs": "^1.11.18", "fs-extra": "^11.3.0", "glob": "^11.0.3", - "lodash": "^4.17.21", + "lodash.merge": "^4.6.2", + "lodash.padstart": "^4.6.1", "pako": "^2.1.0", "socks-proxy-agent": "^8.0.5", "winston": "^3.17.0", @@ -5505,6 +5529,14 @@ "timer-node": "^5.0.9" } }, + "node_modules/epg-grabber/node_modules/curl-generator": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/curl-generator/-/curl-generator-0.4.2.tgz", + "integrity": "sha512-YD74vaPyL46XYNbyRCJV91EhYGDrE/EBiW0X/NUIrNZ23jD0Uwr/6vMrCmobYi5KrjtrqN4SnmMhQNYh3qaULw==", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/epg-parser": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/epg-parser/-/epg-parser-0.3.1.tgz", @@ -6770,6 +6802,28 @@ "node": ">=8" } }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "engines": { + "node": ">=4" + } + }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -8651,6 +8705,11 @@ "integrity": "sha512-T0rZxKmghOOf5YPnn8EY5iLYeWCpZq8G41FfqoVHH5QDTAFaghJRmAdLiadEDq+ztgM2q5PjA+Z1fOwGrLgmtg==", "license": "MIT" }, + "node_modules/lodash.padstart": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.padstart/-/lodash.padstart-4.6.1.tgz", + "integrity": "sha512-sW73O6S8+Tg66eY56DBk85aQzzUJDtpoXFBgELMd5P/SotAguo+1kYO6RuYgXxA4HJH3LFTFPASX6ET6bjfriw==" + }, "node_modules/lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", @@ -13586,6 +13645,14 @@ "@types/lodash": "*" } }, + "@types/lodash.padstart": { + "version": "4.6.9", + "resolved": "https://registry.npmjs.org/@types/lodash.padstart/-/lodash.padstart-4.6.9.tgz", + "integrity": "sha512-KVXQ65AiorTc+Dn9eSRZDs1SnzXULRJcMYhCDEIgsRtHU7mbVpghPSxkySh3Vgm+doWVzpJCA24259fkRL46sA==", + "requires": { + "@types/lodash": "*" + } + }, "@types/lodash.sortby": { "version": "4.7.9", "resolved": "https://registry.npmjs.org/@types/lodash.sortby/-/lodash.sortby-4.7.9.tgz", @@ -14090,6 +14157,15 @@ "http-cookie-agent": "^7.0.2" } }, + "axios-mock-adapter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-2.1.0.tgz", + "integrity": "sha512-AZUe4OjECGCNNssH8SOdtneiQELsqTsat3SQQCWLPjN436/H+L9AjWfV7bF+Zg/YL9cgbhrz5671hoh+Tbn98w==", + "requires": { + "fast-deep-equal": "^3.1.3", + "is-buffer": "^2.0.5" + } + }, "babel-jest": { "version": "30.0.5", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.0.5.tgz", @@ -14741,9 +14817,9 @@ "integrity": "sha512-yi1x3EAWKjQTreYWeSd98431AV+IEE0qoDyOoaHJ7KJ21gv6HtBXHVLX74opVSGqcR8/AbjJBHAHpcOy2bj5Gg==" }, "curl-generator": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/curl-generator/-/curl-generator-0.4.2.tgz", - "integrity": "sha512-YD74vaPyL46XYNbyRCJV91EhYGDrE/EBiW0X/NUIrNZ23jD0Uwr/6vMrCmobYi5KrjtrqN4SnmMhQNYh3qaULw==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/curl-generator/-/curl-generator-0.5.0.tgz", + "integrity": "sha512-dKmp63AJfNSplajvPoWIRfBOmp0IT8NETJ9sFw9IABEUTiJqtXdJx09mXxUlTgVCZDYWVxkCPds+mO9puf1J+w==", "requires": { "ms": "^2.0.0" } @@ -14923,17 +14999,19 @@ "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==" }, "epg-grabber": { - "version": "0.44.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.44.0.tgz", - "integrity": "sha512-M4k/PG1/OIbHV7p8rM23yDWig6WwtpB/LafRzealc8A50HDOGskmkvuhzxf1S34Oe8xL3aU529oW2vocrpijuA==", + "version": "0.45.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.45.0.tgz", + "integrity": "sha512-GqjXRYOJcC3mX9OYdZHlpNEhwBbCF256uKGPXdUodUjkXTmraolIEXYRUeQJmPbL5/do766EewNwIBXqLzHkJA==", "requires": { "@freearhey/core": "^0.14.0", "@types/bluebird": "^3.5.42", "@types/fs-extra": "^11.0.4", - "@types/lodash": "^4.17.20", + "@types/lodash.merge": "^4.6.9", + "@types/lodash.padstart": "^4.6.9", "@types/pako": "^2.0.4", "axios": "^1.12.2", "axios-cache-interceptor": "^1.8.3", + "axios-mock-adapter": "^2.1.0", "bluebird": "^3.7.2", "commander": "^14.0.0", "curl-generator": "^0.4.2", @@ -14941,7 +15019,8 @@ "dayjs": "^1.11.18", "fs-extra": "^11.3.0", "glob": "^11.0.3", - "lodash": "^4.17.21", + "lodash.merge": "^4.6.2", + "lodash.padstart": "^4.6.1", "pako": "^2.1.0", "socks-proxy-agent": "^8.0.5", "winston": "^3.17.0", @@ -14965,6 +15044,14 @@ "pako": "^2.1.0", "timer-node": "^5.0.9" } + }, + "curl-generator": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/curl-generator/-/curl-generator-0.4.2.tgz", + "integrity": "sha512-YD74vaPyL46XYNbyRCJV91EhYGDrE/EBiW0X/NUIrNZ23jD0Uwr/6vMrCmobYi5KrjtrqN4SnmMhQNYh3qaULw==", + "requires": { + "ms": "^2.0.0" + } } } }, @@ -15804,6 +15891,11 @@ "binary-extensions": "^2.0.0" } }, + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, "is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -17201,6 +17293,11 @@ "resolved": "https://registry.npmjs.org/lodash.orderby/-/lodash.orderby-4.6.0.tgz", "integrity": "sha512-T0rZxKmghOOf5YPnn8EY5iLYeWCpZq8G41FfqoVHH5QDTAFaghJRmAdLiadEDq+ztgM2q5PjA+Z1fOwGrLgmtg==" }, + "lodash.padstart": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.padstart/-/lodash.padstart-4.6.1.tgz", + "integrity": "sha512-sW73O6S8+Tg66eY56DBk85aQzzUJDtpoXFBgELMd5P/SotAguo+1kYO6RuYgXxA4HJH3LFTFPASX6ET6bjfriw==" + }, "lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", diff --git a/package.json b/package.json index ef31e202..e58062a9 100644 --- a/package.json +++ b/package.json @@ -78,9 +78,10 @@ "consola": "^3.4.2", "cross-env": "^10.0.0", "csv-parser": "^3.2.0", + "curl-generator": "^0.5.0", "cwait": "^1.1.2", "dayjs": "^1.11.13", - "epg-grabber": "^0.44.0", + "epg-grabber": "^0.45.0", "epg-parser": "^0.3.1", "eslint": "^9.32.0", "eslint-config-prettier": "^10.1.8", From 5845b2477875f16224a0d194b792bc3cddc3111e Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Wed, 29 Oct 2025 07:56:55 +0300 Subject: [PATCH 06/11] Update grab.test.ts --- tests/commands/epg/grab.test.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/commands/epg/grab.test.ts b/tests/commands/epg/grab.test.ts index afb3ddf0..8c311127 100644 --- a/tests/commands/epg/grab.test.ts +++ b/tests/commands/epg/grab.test.ts @@ -99,16 +99,24 @@ describe('epg:grab', () => { content('tests/__data__/expected/epg_grab/gzip.guide.xml') ) - const expected = pako.ungzip(fs.readFileSync('tests/__data__/output/guides/gzip.guide.xml.gz')) - const result = pako.ungzip( - fs.readFileSync('tests/__data__/expected/epg_grab/gzip.guide.xml.gz') + const outputString = pako.ungzip( + fs.readFileSync('tests/__data__/output/guides/gzip.guide.xml.gz'), + { to: 'string' } ) - expect(expected).toEqual(result) + const expectedString = pako.ungzip( + fs.readFileSync('tests/__data__/expected/epg_grab/gzip.guide.xml.gz'), + { to: 'string' } + ) + + const output = new Set(outputString.split('\r\n')) + const expected = new Set(expectedString.split('\r\n')) + + expect(output).toEqual(expected) }) }) function content(filepath: string) { - return fs.readFileSync(pathToFileURL(filepath), { - encoding: 'utf8' - }) + const string = fs.readFileSync(pathToFileURL(filepath), 'utf8') + + return new Set(string.split('\r\n')) } From f07af88a8cc58bddfdf4863146281259720b4373 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Wed, 29 Oct 2025 07:58:58 +0300 Subject: [PATCH 07/11] Update scripts --- scripts/commands/epg/grab.ts | 153 +++++++++++++++++++++-------------- scripts/core/index.ts | 1 - scripts/core/siteConfig.ts | 71 ---------------- scripts/default.config.js | 10 +++ scripts/types/queue.d.ts | 4 +- 5 files changed, 105 insertions(+), 134 deletions(-) delete mode 100644 scripts/core/siteConfig.ts create mode 100644 scripts/default.config.js diff --git a/scripts/commands/epg/grab.ts b/scripts/commands/epg/grab.ts index 0b90bc19..c111a2bf 100644 --- a/scripts/commands/epg/grab.ts +++ b/scripts/commands/epg/grab.ts @@ -1,15 +1,19 @@ import { Logger, Timer, Collection, Template } from '@freearhey/core' import epgGrabber, { EPGGrabber, EPGGrabberMock } from 'epg-grabber' -import { loadJs, parseProxy, SiteConfig, Queue } from '../../core' +import { CurlBody } from 'curl-generator/dist/bodies/body' +import { loadJs, parseProxy, Queue } from '../../core' import { Channel, Guide, Program } from '../../models' import { SocksProxyAgent } from 'socks-proxy-agent' +import defaultConfig from '../../default.config' import { PromisyClass, TaskQueue } from 'cwait' import { Storage } from '@freearhey/storage-js' +import { CurlGenerator } from 'curl-generator' import { QueueItem } from '../../types/queue' import { Option, program } from 'commander' import { SITES_DIR } from '../../constants' import { data, loadData } from '../../api' import dayjs, { Dayjs } from 'dayjs' +import merge from 'lodash.merge' import path from 'path' program @@ -53,6 +57,7 @@ program .env('GZIP') ) .addOption(new Option('--curl', 'Display each request as CURL').default(false).env('CURL')) + .addOption(new Option('--debug', 'Enable debug mode').default(false).env('DEBUG')) .parse() interface GrabOptions { @@ -61,6 +66,7 @@ interface GrabOptions { output: string gzip: boolean curl: boolean + debug: boolean maxConnections: number timeout?: number delay?: number @@ -72,25 +78,87 @@ interface GrabOptions { const options: GrabOptions = program.opts() async function main() { - if (!options.site && !options.channels) + if (typeof options.site !== 'string' && typeof options.channels !== 'string') throw new Error('One of the arguments must be presented: `--site` or `--channels`') - const logger = new Logger() + const LOG_LEVELS = { info: 3, debug: 4 } + const logger = new Logger({ level: options.debug ? LOG_LEVELS['debug'] : LOG_LEVELS['info'] }) logger.info('starting...') + let config: epgGrabber.Types.SiteConfig = defaultConfig - logger.info('config:') - logger.tree(options) + if (typeof options.timeout === 'number') + config = merge(config, { request: { timeout: options.timeout } }) + if (options.proxy !== undefined) { + const proxy = parseProxy(options.proxy) + if ( + proxy.protocol && + ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol)) + ) { + const socksProxyAgent = new SocksProxyAgent(options.proxy) + config = merge(config, { + request: { httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } + }) + } else { + config = merge(config, { request: { proxy } }) + } + } + + if (typeof options.output === 'string') config.output = options.output + if (typeof options.days === 'number') config.days = options.days + if (typeof options.delay === 'number') config.delay = options.delay + if (typeof options.maxConnections === 'number') config.maxConnections = options.maxConnections + if (typeof options.curl === 'boolean') config.curl = options.curl + if (typeof options.gzip === 'boolean') config.gzip = options.gzip + + const grabber = + process.env.NODE_ENV === 'test' ? new EPGGrabberMock(config) : new EPGGrabber(config) + + const globalConfig = grabber.globalConfig + + logger.debug(`config: ${JSON.stringify(globalConfig, null, 2)}`) + + grabber.client.instance.interceptors.request.use( + request => { + if (globalConfig.curl) { + type AllowedMethods = + | 'GET' + | 'get' + | 'POST' + | 'post' + | 'PUT' + | 'put' + | 'PATCH' + | 'patch' + | 'DELETE' + | 'delete' + + const url = request.url || '' + const method = request.method ? (request.method as AllowedMethods) : 'GET' + const headers = request.headers + ? (request.headers.toJSON() as Record) + : undefined + const body = request.data ? (request.data as CurlBody) : undefined + + const curl = CurlGenerator({ url, method, headers, body }) + + console.log(curl) + } + + return request + }, + error => Promise.reject(error) + ) logger.info('loading channels...') const storage = new Storage() let files: string[] = [] - if (options.site) { + if (typeof options.site === 'string') { let pattern = path.join(SITES_DIR, options.site, '*.channels.xml') pattern = pattern.replace(/\\/g, '/') files = await storage.list(pattern) - } else if (options.channels) { + } else if (typeof options.channels === 'string') { files = await storage.list(options.channels) } @@ -105,7 +173,7 @@ async function main() { channelsFromXML.concat(_channelsFromXML) } - if (options.lang) { + if (typeof options.lang === 'string') { channelsFromXML = channelsFromXML.filter((channel: Channel) => { if (!options.lang) return true @@ -119,7 +187,6 @@ async function main() { await loadData() logger.info('creating queue...') - let index = 0 const queue = new Queue() @@ -127,40 +194,13 @@ async function main() { channel.index = index++ if (!channel.site || !channel.site_id || !channel.name) continue - const configObject = await loadJs(channel.getConfigPath()) - - const siteConfig = new SiteConfig(configObject) - - siteConfig.filepath = channel.getConfigPath() - - if (typeof options.timeout === 'number') { - siteConfig.request = { ...siteConfig.request, ...{ timeout: options.timeout } } - } - if (typeof options.days === 'number') siteConfig.days = options.days - if (typeof options.delay === 'number') siteConfig.delay = options.delay - if (typeof options.curl === 'boolean') siteConfig.curl = options.curl - if (typeof options.proxy === 'string') { - const proxy = parseProxy(options.proxy) - - if ( - proxy.protocol && - ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol)) - ) { - const socksProxyAgent = new SocksProxyAgent(options.proxy) - - siteConfig.request = { - ...siteConfig.request, - ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } - } - } else { - siteConfig.request = { ...siteConfig.request, ...{ proxy } } - } - } + const config = await loadJs(channel.getConfigPath()) + const days: number = config.days || globalConfig.days if (!channel.xmltv_id) channel.xmltv_id = channel.site_id const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString()) - const dates = Array.from({ length: siteConfig.days }, (_, day) => currDate.add(day, 'd')) + const dates = Array.from({ length: days }, (_, day) => currDate.add(day, 'd')) dates.forEach((date: Dayjs) => { const key = `${channel.site}:${channel.lang}:${channel.xmltv_id}:${date.toJSON()}` @@ -168,14 +208,12 @@ async function main() { queue.add(key, { channel, date, - siteConfig, + config, error: null }) }) } - const grabber = process.env.NODE_ENV === 'test' ? new EPGGrabberMock() : new EPGGrabber() - const taskQueue = new TaskQueue(Promise as PromisyClass, options.maxConnections) const queueItems = queue.getItems() @@ -188,10 +226,10 @@ async function main() { const requests = queueItems.map( taskQueue.wrap(async (queueItem: QueueItem) => { - const { channel, siteConfig, date } = queueItem + const { channel, config, date } = queueItem if (!channel.logo) { - if (siteConfig.logo) { + if (config.logo) { channel.logo = await grabber.loadLogo(channel, date) } else { channel.logo = getLogoForChannel(channel) @@ -203,7 +241,7 @@ async function main() { const channelPrograms = await grabber.grab( channel, date, - siteConfig, + config, (context: epgGrabber.Types.GrabCallbackContext, error: Error | null) => { logger.info( ` [${i}/${total}] ${context.channel.site} (${context.channel.lang}) - ${ @@ -235,23 +273,18 @@ async function main() { const pathTemplate = new Template(options.output) - const channelsGroupedByKey = channels - .sortBy([(channel: Channel) => channel.index, (channel: Channel) => channel.xmltv_id]) - .uniqBy((channel: Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`) - .groupBy((channel: Channel) => { - return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' }) - }) + const channelsGroupedByKey = channels.groupBy((channel: Channel) => { + return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' }) + }) - const programsGroupedByKey = programs - .sortBy([(program: Program) => program.channel, (program: Program) => program.start]) - .groupBy((program: Program) => { - const lang = - program.titles && program.titles.length && program.titles[0].lang - ? program.titles[0].lang - : 'en' + const programsGroupedByKey = programs.groupBy((program: Program) => { + const lang = + program.titles && program.titles.length && program.titles[0].lang + ? program.titles[0].lang + : 'en' - return pathTemplate.format({ lang, site: program.site || '' }) - }) + return pathTemplate.format({ lang, site: program.site || '' }) + }) for (const groupKey of channelsGroupedByKey.keys()) { const groupChannels = new Collection(channelsGroupedByKey.get(groupKey)) diff --git a/scripts/core/index.ts b/scripts/core/index.ts index 6d69e053..fbe32262 100644 --- a/scripts/core/index.ts +++ b/scripts/core/index.ts @@ -1,4 +1,3 @@ export * from './htmlTable' -export * from './siteConfig' export * from './utils' export * from './queue' diff --git a/scripts/core/siteConfig.ts b/scripts/core/siteConfig.ts deleted file mode 100644 index a69c3abc..00000000 --- a/scripts/core/siteConfig.ts +++ /dev/null @@ -1,71 +0,0 @@ -import * as epgGrabber from 'epg-grabber' -import merge from 'lodash.merge' - -const _default = { - days: 1, - delay: 0, - output: 'guide.xml', - request: { - method: 'GET', - maxContentLength: 5242880, - timeout: 30000, - withCredentials: true, - jar: null, - responseType: 'arraybuffer', - cache: false, - headers: null, - data: null - }, - maxConnections: 1, - site: undefined, - url: undefined, - parser: undefined, - channels: undefined, - lang: 'en', - debug: false, - gzip: false, - curl: false, - logo: '' -} - -export class SiteConfig { - days: number - lang: string - delay: number - debug: boolean - gzip: boolean - curl: boolean - maxConnections: number - output: string - request: epgGrabber.Types.SiteConfigRequestConfig - site: string - channels?: string | string[] - url: ((context: epgGrabber.Types.SiteConfigRequestContext) => string | Promise) | string - parser: ( - context: epgGrabber.Types.SiteConfigParserContext - ) => - | epgGrabber.Types.SiteConfigParserResult[] - | Promise - logo: ((context: epgGrabber.Types.SiteConfigRequestContext) => string | Promise) | string - filepath: string - - constructor(config: epgGrabber.Types.SiteConfigObject) { - this.site = config.site - this.channels = config.channels - this.url = config.url - this.parser = config.parser - this.filepath = config.filepath - - this.days = config.days || _default.days - this.lang = config.lang || _default.lang - this.delay = config.delay || _default.delay - this.debug = config.debug || _default.debug - this.maxConnections = config.maxConnections || _default.maxConnections - this.gzip = config.gzip || _default.gzip - this.curl = config.curl || _default.curl - this.output = config.output || _default.output - this.logo = config.logo || _default.logo - - this.request = merge(_default.request, config.request) - } -} diff --git a/scripts/default.config.js b/scripts/default.config.js new file mode 100644 index 00000000..98b39982 --- /dev/null +++ b/scripts/default.config.js @@ -0,0 +1,10 @@ +export default { + days: 1, + delay: 0, + request: { + maxContentLength: 5242880, + timeout: 30000, + withCredentials: true, + jar: null + } +} diff --git a/scripts/types/queue.d.ts b/scripts/types/queue.d.ts index 26615c15..ef0fead8 100644 --- a/scripts/types/queue.d.ts +++ b/scripts/types/queue.d.ts @@ -1,10 +1,10 @@ -import { SiteConfig } from '../core/siteConfig' import { Channel } from '../models/channel' +import epgGrabber from 'epg-grabber' import { Dayjs } from 'dayjs' export interface QueueItem { channel: Channel date: Dayjs - siteConfig: SiteConfig + config: epgGrabber.Types.SiteConfig error: string | null } From 8a7b682205c243a77388db0c1a65a06c6af683bd Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Wed, 29 Oct 2025 07:59:17 +0300 Subject: [PATCH 08/11] Update CONTRIBUTING.md --- CONTRIBUTING.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f5e9586b..e7089e05 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,9 +66,16 @@ module.exports = { } ``` -### Context Object +### Request Context Object -From each function in `config.js` you can access a `context` object containing the following data: +Inside `url()`, `logo()`, `request.data()`, `request.headers()` functions in `*.config.js` you can access a `context` object containing the following data: + +- `channel`: The object describing the current channel (xmltv_id, site_id, name, lang) +- `date`: The 'dayjs' instance with the requested date + +## Parser Context Object + +Inside `parser()` function in `*.config.js` you can access a `context` object containing the following data: - `channel`: The object describing the current channel (xmltv_id, site_id, name, lang) - `date`: The 'dayjs' instance with the requested date @@ -427,7 +434,7 @@ To run scripts use the `npm run ` command. - `api:generate`: generates a JSON file with all channels for the [iptv-org/api](https://github.com/iptv-org/api) repository. - `channels:lint`: сhecks the channel lists for syntax errors. - `channels:parse`: generates a list of channels based on the site configuration. -- `channels:parse`: formats `*.channels.xml` files. The process involves removing invalid `xmltv_id`, adding missing Feed ID, and sorting the list. +- `channels:format`: formats `*.channels.xml` files. The process involves removing invalid `xmltv_id`, adding missing Feed ID, and sorting the list. - `channels:edit`: utility for quick channels mapping. - `channels:validate`: checks the description of channels for errors. - `sites:init`: creates a new site config from the template. From 0f89f97e0b867743089244d200751c46416211a3 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Thu, 30 Oct 2025 07:39:09 +0300 Subject: [PATCH 09/11] Update default.config.js --- scripts/default.config.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/default.config.js b/scripts/default.config.js index 98b39982..2bc576ba 100644 --- a/scripts/default.config.js +++ b/scripts/default.config.js @@ -1,10 +1,10 @@ -export default { - days: 1, - delay: 0, - request: { - maxContentLength: 5242880, - timeout: 30000, - withCredentials: true, - jar: null - } -} +export default { + days: 1, + delay: 0, + request: { + maxContentLength: 5242880, + timeout: 30000, + withCredentials: true, + jar: null + } +} From 6641d23714c0134a27df726ce800d010d74a7762 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Thu, 30 Oct 2025 07:39:13 +0300 Subject: [PATCH 10/11] Update utils.ts --- scripts/core/utils.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/core/utils.ts b/scripts/core/utils.ts index 23d2145e..ae2377ac 100644 --- a/scripts/core/utils.ts +++ b/scripts/core/utils.ts @@ -104,3 +104,7 @@ export async function loadIssues(props?: { labels: string[] | string }) { return new Collection(issues).map(data => new Issue(data)) } + +export function parseNumber(value: string): number { + return parseInt(value) +} From 530db7dafa5763f68e70a67a8d4f618c02a0d6a0 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Thu, 30 Oct 2025 07:39:21 +0300 Subject: [PATCH 11/11] Update grab.ts --- scripts/commands/epg/grab.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/commands/epg/grab.ts b/scripts/commands/epg/grab.ts index c111a2bf..f61d7de5 100644 --- a/scripts/commands/epg/grab.ts +++ b/scripts/commands/epg/grab.ts @@ -1,7 +1,7 @@ import { Logger, Timer, Collection, Template } from '@freearhey/core' import epgGrabber, { EPGGrabber, EPGGrabberMock } from 'epg-grabber' +import { loadJs, parseProxy, Queue, parseNumber } from '../../core' import { CurlBody } from 'curl-generator/dist/bodies/body' -import { loadJs, parseProxy, Queue } from '../../core' import { Channel, Guide, Program } from '../../models' import { SocksProxyAgent } from 'socks-proxy-agent' import defaultConfig from '../../default.config' @@ -29,12 +29,12 @@ program .addOption( new Option('-t, --timeout ', 'Override the default timeout for each request') .env('TIMEOUT') - .argParser(parseInt) + .argParser(parseNumber) ) .addOption( new Option('-d, --delay ', 'Override the default delay between request') .env('DELAY') - .argParser(parseInt) + .argParser(parseNumber) ) .addOption(new Option('-x, --proxy ', 'Use the specified proxy').env('PROXY')) .addOption( @@ -42,13 +42,13 @@ program '--days ', 'Override the number of days for which the program will be loaded (defaults to the value from the site config)' ) - .argParser(parseInt) + .argParser(parseNumber) .env('DAYS') ) .addOption( new Option('--maxConnections ', 'Limit on the number of concurrent requests') .default(1) - .argParser(parseInt) + .argParser(parseNumber) .env('MAX_CONNECTIONS') ) .addOption(