diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 2f504e4ef..619075213 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -45,6 +45,8 @@ jobs: git commit -m "[Bot] Update ${FILE}" -m "Committed by [iptv-bot](https://github.com/apps/iptv-bot) via [update](https://github.com/iptv-org/epg/actions/runs/${{ github.run_id }}) workflow." --no-verify git push fi + - name: load workers + run: npm run workers:load - name: update guides.md run: npm run guides:update - run: git status @@ -59,7 +61,7 @@ jobs: git push fi - name: generate .api/guides.json - run: npm run api:generate + run: npm run guides:export - run: git status - name: move .api/guides.json to iptv-org/api uses: JamesIves/github-pages-deploy-action@4.1.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ee3bec41..5920c61f2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -422,6 +422,19 @@ To do this, you just need to place the `worker.json` file in a public directory } ``` +If you generate a guide in multiple formats, you can specify the paths to all the files as follows: + +```json +{ + "channels": "path/to/channels.xml", + "guide": { + "xml": "path/to/guide.xml", + "gzip": "path/to/guide.xml.gz", + "json": "path/to/guide.json" + } +} +``` + And then add server domain or IP address to the [workers.txt](workers.txt) file. @@ -454,7 +467,6 @@ To run scripts use the `npm run ` command. - `act:check`: allows to test the [check](https://github.com/iptv-org/iptv/blob/master/.github/workflows/check.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act). - `act:update`: allows to test the [update](https://github.com/iptv-org/iptv/blob/master/.github/workflows/update.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act). - `api:load`: downloads the latest channels data from the [iptv-org/api](https://github.com/iptv-org/api). -- `api:generate`: generates a JSON file with all channels for the [iptv-org/api](https://github.com/iptv-org/api) repository. - `channels:lint`: сhecks the channel lists for syntax errors. - `channels:parse`: generates a list of channels based on the site configuration. - `channels:format`: formats `*.channels.xml` files. The process involves removing invalid `xmltv_id`, adding missing Feed ID, and sorting the list. @@ -463,6 +475,8 @@ To run scripts use the `npm run ` command. - `sites:init`: creates a new site config from the template. - `sites:update`: updates the list of sites and their status in [SITES.md](SITES.md). - `guides:update`: updates the list of guides and their status in [GUIDES.md](GUIDES.md). +- `guides:export`: exports a list of all guides in JSON format for the [iptv-org/api](https://github.com/iptv-org/api) repository. +- `workers:load`: loads a list of all workers and their configurations. - `grab`: downloads a program from a specified source. - `serve`: starts the [web server](https://github.com/vercel/serve). - `lint`: сhecks the scripts for syntax errors. diff --git a/README.md b/README.md index cd59eb8c9..499bce01b 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Usage: npm run grab --- [options] Options: -s, --sites A comma-separated list of the sites to parse - -c, --channels Path to *.channels.xml file (required if the "--site" attribute is + -c, --channels Path to *.channels.xml file (required if the "--sites" attribute is not specified) -o, --output Path to output file (default: "guide.xml") -l, --lang Allows you to restrict downloading to channels in specified languages only (example: "en,id") @@ -64,7 +64,8 @@ Options: -x, --proxy Use the specified proxy (example: "socks5://username:password@127.0.0.1:1234") --days Number of days for which the program will be loaded (defaults to the value from the site config) --maxConnections Number of concurrent requests (default: 1) - --gzip Specifies whether or not to create a compressed version of the guide (default: false) + --gzip [path] Specifies whether or not to create a compressed version of the guide (default: false) + --json [path] Specifies whether or not to create a JSON version of the guide (default: false) --curl Display each request as CURL (default: false) ``` @@ -197,11 +198,13 @@ docker run \ -e CRON_SCHEDULE="0 0,12 * * *" \ -e MAX_CONNECTIONS=10 \ -e GZIP=true \ +-e JSON=true \ -e CURL=true \ -e PROXY="socks5://127.0.0.1:1234" \ -e DAYS=14 \ -e TIMEOUT=5 \ -e DELAY=2 \ +-e RUN_AT_STARTUP=true \ ghcr.io/iptv-org/epg:master ``` @@ -210,6 +213,7 @@ ghcr.io/iptv-org/epg:master | CRON_SCHEDULE | A [cron expression](https://crontab.guru/) describing the schedule of the guide loadings (default: "0 0 \* \* \*") | | MAX_CONNECTIONS | Limit on the number of concurrent requests (default: 1) | | GZIP | Boolean value indicating whether to create a compressed version of the guide (default: false) | +| JSON | Boolean value indicating whether to create a JSON version of the guide (default: false) | | CURL | Display each request as CURL (default: false) | | PROXY | Use the specified proxy | | DAYS | Number of days for which the guide will be loaded (defaults to the value from the site config) | diff --git a/package-lock.json b/package-lock.json index f66bf79ce..dcddf5890 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,7 +44,7 @@ "csv-parser": "^3.2.0", "curl-generator": "^0.5.0", "dayjs": "^1.11.20", - "epg-grabber": "^0.46.2", + "epg-grabber": "^0.47.0", "epg-parser": "^0.5.0", "eslint": "^10.2.1", "eslint-config-prettier": "^10.1.8", @@ -5632,9 +5632,9 @@ } }, "node_modules/epg-grabber": { - "version": "0.46.2", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.46.2.tgz", - "integrity": "sha512-pcZD9eo0d5QFqLr/YUXh4SapjrE3MfHvjyIokVpfCVI5wVXkdy3q9SxM8b34hPYPy2caeaCTpZM1yTAeoIOIMw==", + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.47.0.tgz", + "integrity": "sha512-kl+erC90n1ipt2aThPF9o7bkxYsS/QIUQ56M/RWfOL9vyKZxSU9vvzMDp88HwCZ6NWMYPpeDTooA1XnsWT86Eg==", "license": "MIT", "dependencies": { "@freearhey/core": "^0.14.0", @@ -14952,9 +14952,9 @@ "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==" }, "epg-grabber": { - "version": "0.46.2", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.46.2.tgz", - "integrity": "sha512-pcZD9eo0d5QFqLr/YUXh4SapjrE3MfHvjyIokVpfCVI5wVXkdy3q9SxM8b34hPYPy2caeaCTpZM1yTAeoIOIMw==", + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.47.0.tgz", + "integrity": "sha512-kl+erC90n1ipt2aThPF9o7bkxYsS/QIUQ56M/RWfOL9vyKZxSU9vvzMDp88HwCZ6NWMYPpeDTooA1XnsWT86Eg==", "requires": { "@freearhey/core": "^0.14.0", "@types/bluebird": "^3.5.42", diff --git a/package.json b/package.json index 017530196..088c52191 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,6 @@ "act:format": "act workflow_dispatch -W .github/workflows/format.yml", "act:update": "act workflow_dispatch -W .github/workflows/update.yml", "api:load": "tsx scripts/commands/api/load.ts", - "api:generate": "tsx scripts/commands/api/generate.ts", "channels:lint": "tsx scripts/commands/channels/lint.mts", "channels:parse": "tsx scripts/commands/channels/parse.ts", "channels:format": "tsx scripts/commands/channels/format.ts", @@ -14,6 +13,8 @@ "sites:init": "tsx scripts/commands/sites/init.ts", "sites:update": "tsx scripts/commands/sites/update.ts", "guides:update": "tsx scripts/commands/guides/update.ts", + "guides:export": "tsx scripts/commands/guides/export.ts", + "workers:load": "tsx scripts/commands/workers/load.ts", "grab": "tsx scripts/commands/epg/grab.ts", "lint": "npx eslint \"{scripts,tests,sites}/**/*.{ts,mts,js}\"", "test": "cross-env TZ=Pacific/Nauru npx jest --runInBand", @@ -72,7 +73,7 @@ "csv-parser": "^3.2.0", "curl-generator": "^0.5.0", "dayjs": "^1.11.20", - "epg-grabber": "^0.46.2", + "epg-grabber": "^0.47.0", "epg-parser": "^0.5.0", "eslint": "^10.2.1", "eslint-config-prettier": "^10.1.8", diff --git a/scripts/commands/api/generate.ts b/scripts/commands/api/generate.ts deleted file mode 100644 index 083113c58..000000000 --- a/scripts/commands/api/generate.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { ChannelGuideObject } from '../../types/channel' -import { SITES_DIR, API_DIR } from '../../constants' -import { Logger, Collection } from '@freearhey/core' -import epgGrabber, { EPGGrabber } from 'epg-grabber' -import { Storage } from '@freearhey/storage-js' -import { Channel } from '../../models' -import path from 'path' - -async function main() { - const logger = new Logger() - - logger.start('staring...') - - logger.info('loading channels...') - const sitesStorage = new Storage(SITES_DIR) - - const files: string[] = await sitesStorage.list('**/*.channels.xml') - - const channels = new Collection() - for (const filepath of files) { - const xml = await sitesStorage.load(filepath) - const parsedChannels = EPGGrabber.parseChannelsXML(xml) - const channelsFromXML = new Collection(parsedChannels).map( - (channel: epgGrabber.Channel) => new Channel(channel.toObject()) - ) - - channelsFromXML.forEach((channel: Channel) => { - channels.add(channel) - }) - } - - logger.info(`found ${channels.count()} channel(s)`) - - const output = channels.map((channel: Channel) => channel.getGuideObject()) - - const apiStorage = new Storage(API_DIR) - const outputFilename = 'guides.json' - await apiStorage.save('guides.json', output.toJSON()) - - logger.info(`saved to "${path.join(API_DIR, outputFilename)}"`) -} - -main() diff --git a/scripts/commands/epg/grab.ts b/scripts/commands/epg/grab.ts index 2dabb6b53..eb9ee55e3 100644 --- a/scripts/commands/epg/grab.ts +++ b/scripts/commands/epg/grab.ts @@ -54,7 +54,10 @@ program .argParser(parseNumber) .env('MAX_CONNECTIONS') ) - .addOption(new Option('--gzip', 'Create a compressed version of the guide as well').env('GZIP')) + .addOption( + new Option('--gzip [path]', 'Create a compressed version of the guide as well').env('GZIP') + ) + .addOption(new Option('--json [path]', 'Create a JSON version of the guide as well').env('JSON')) .addOption(new Option('--curl', 'Display each request as CURL').env('CURL')) .addOption(new Option('--debug', 'Enable debug mode').env('DEBUG')) .parse() @@ -63,7 +66,8 @@ interface GrabOptions { sites?: string[] channels?: string output?: string - gzip?: boolean + gzip?: boolean | string + json?: boolean | string curl?: boolean debug?: boolean maxConnections?: number @@ -109,7 +113,10 @@ async function main() { if (typeof options.maxConnections === 'number') globalConfig.maxConnections = options.maxConnections if (typeof options.curl === 'boolean') globalConfig.curl = options.curl - if (typeof options.gzip === 'boolean') globalConfig.gzip = options.gzip + if (typeof options.gzip === 'boolean' || typeof options.gzip === 'string') + globalConfig.gzip = options.gzip + if (typeof options.json === 'boolean' || typeof options.json === 'string') + globalConfig.json = options.json if (typeof options.debug === 'boolean') globalConfig.debug = options.debug logger.debug(`config: ${JSON.stringify(globalConfig, null, 2)}`) @@ -298,6 +305,7 @@ async function main() { }) const gzip = globalConfig.gzip || defaultConfig.gzip + const json = globalConfig.json || defaultConfig.json for (const groupKey of channelsGroupedByKey.keys()) { const groupChannels = new Collection(channelsGroupedByKey.get(groupKey)) @@ -305,6 +313,7 @@ async function main() { const guide = new Guide({ filepath: groupKey, gzip, + json, channels: groupChannels, programs: groupPrograms }) diff --git a/scripts/commands/guides/export.ts b/scripts/commands/guides/export.ts new file mode 100644 index 000000000..5f01f04f7 --- /dev/null +++ b/scripts/commands/guides/export.ts @@ -0,0 +1,88 @@ +import { SITES_DIR, API_DIR, DATA_DIR } from '../../constants' +import { Logger, Collection } from '@freearhey/core' +import epgGrabber, { EPGGrabber } from 'epg-grabber' +import { Storage } from '@freearhey/storage-js' +import { Channel, Worker, WorkerData } from '../../models' +import path from 'path' +import { ApiGuide } from '../../models/apiGuide' + +async function main() { + const logger = new Logger() + + logger.start('staring...') + + logger.info('loading channels...') + const channels = await loadChannels() + logger.info(`found ${channels.count()} channel(s)`) + + logger.info('loading workers.json...') + const workers = await loadWorkers() + const guidesMap = workers.reduce((acc, data: WorkerData) => { + const worker = new Worker(data) + if (!worker.channels) return acc + worker.channels.forEach(channel => { + const [channelId, feedId] = channel.xmltv_id.split('@') + const key = [channelId, feedId, channel.site, channel.site_id, channel.lang].join('_') + acc[key] = worker.getGuideSources() + }) + return acc + }, {}) + + logger.info('preparing output...') + const output = channels.map((channel: Channel) => { + const [channelId, feedId] = channel.xmltv_id.split('@') + const key = [channelId, feedId, channel.site, channel.site_id, channel.lang].join('_') + + return new ApiGuide({ + channel: channelId, + feed: feedId, + site: channel.site, + site_id: channel.site_id, + site_name: channel.name, + lang: channel.lang, + sources: guidesMap[key] || [] + }) + }) + + logger.info('saving output...') + const outputFilename = 'guides.json' + await saveOutput(outputFilename, output.toJSON()) + + logger.info(`saved to "${path.join(API_DIR, outputFilename)}"`) +} + +main() + +async function saveOutput(filename: string, json: string) { + const apiStorage = new Storage(API_DIR) + await apiStorage.save(filename, json) +} + +async function loadWorkers() { + const dataStorage = new Storage(DATA_DIR) + + const workers = await dataStorage.json('workers.json') + + return Array.isArray(workers) ? workers : [] +} + +async function loadChannels() { + const sitesStorage = new Storage(SITES_DIR) + + const files: string[] = await sitesStorage.list('**/*.channels.xml') + + const channels = new Collection() + for (const filepath of files) { + const xml = await sitesStorage.load(filepath) + const parsedChannels = EPGGrabber.parseChannelsXML(xml) + const channelsFromXML = new Collection(parsedChannels).map( + (channel: epgGrabber.Channel) => new Channel(channel.toObject()) + ) + + channelsFromXML.forEach((channel: Channel) => { + channels.add(channel) + }) + } + + return channels +} diff --git a/scripts/commands/guides/update.ts b/scripts/commands/guides/update.ts index f43b96b6c..cd9e50b04 100644 --- a/scripts/commands/guides/update.ts +++ b/scripts/commands/guides/update.ts @@ -1,114 +1,26 @@ import { HTMLTableRow, HTMLTableDataItem, HTMLTableColumn } from '../../types/htmlTable' -import epgGrabber, { EPGGrabber } from 'epg-grabber' -import AxiosMockAdapter from 'axios-mock-adapter' +import { Worker, WorkerData, WorkerGuideSource } from '../../models' +import { DATA_DIR, ROOT_DIR } from '../../constants' import { Storage } from '@freearhey/storage-js' -import { Channel, Worker } from '../../models' import { Collection } from '@freearhey/core' -import { ROOT_DIR } from '../../constants' import { Logger } from '@freearhey/core' import { HTMLTable } from '../../core' -import epgParser from 'epg-parser' -import axios from 'axios' async function main() { const logger = new Logger({ level: process.env.NODE_ENV === 'test' ? -999 : 3 }) - const rootStorage = new Storage(ROOT_DIR) - const workers = new Map() + const dataStorage = new Storage(DATA_DIR) - logger.info('loading workers.txt...') - const workersTxt = await rootStorage.load('workers.txt') + logger.info('loading workers.json...') + const workers = await dataStorage.json('workers.json') - workersTxt.split('\r\n').forEach((host: string) => { - if (!host) return - - const worker = new Worker({ host }) - - workers.set(host, worker) - }) - - for (const worker of workers.values()) { - logger.info(`processing "${worker.host}"...`) - - const client = axios.create({ - baseURL: worker.getBaseUrl(), - timeout: 60000 - }) - - if (process.env.NODE_ENV === 'test') { - const mock = new AxiosMockAdapter(client) - if (worker.host === 'example.com') { - mock.onGet('worker.json').reply(404) - } else { - const testStorage = new Storage('tests/__data__/input/guides_update') - mock.onGet('worker.json').reply(200, await testStorage.load('worker.json')) - mock.onGet('channels.xml').reply(200, await testStorage.load('channels.xml')) - mock.onGet('guide.xml').reply(200, await testStorage.load('guide.xml')) - } - } - - const workerJson = await client - .get('worker.json') - .then(res => res.data) - .catch(err => { - worker.status = err.status - logger.error(err.message) - }) - - if (!workerJson) { - worker.status = 'MISSING_WORKER_CONFIG' - logger.error('Unable to load "workers.json"') - continue - } - - worker.channelsPath = workerJson.channels - worker.guidePath = workerJson.guide - - if (!worker.channelsPath) { - worker.status = 'MISSING_CHANNELS_PATH' - logger.error('The "channels" property is missing from the workers config') - continue - } - - if (!worker.guidePath) { - worker.status = 'MISSING_GUIDE_PATH' - logger.error('The "guide" property is missing from the workers config') - continue - } - - const channelsXml = await client - .get(worker.channelsPath) - .then(res => res.data) - .catch(err => { - worker.status = err.status - logger.error(err.message) - }) - - if (!channelsXml) continue - - const parsedChannels = EPGGrabber.parseChannelsXML(channelsXml) - worker.channels = new Collection(parsedChannels).map( - (channel: epgGrabber.Channel) => new Channel(channel.toObject()) - ) - - const guideXml = await client - .get(worker.guidePath) - .then(res => res.data) - .catch(err => { - worker.status = err.status - logger.error(err.message) - }) - - if (!guideXml) continue - - const parsedGuide = epgParser.parse(guideXml) - worker.lastUpdated = parsedGuide.date - - worker.status = 'OK' - } + if (!Array.isArray(workers)) return logger.info('creating guides table...') const rows = new Collection() - workers.forEach((worker: Worker) => { + workers.forEach((data: WorkerData) => { + const worker = new Worker(data) + + const sources = worker.getGuideSources() rows.add( new Collection([ { value: worker.host }, @@ -116,16 +28,18 @@ async function main() { { value: worker.getChannelsCount().toString(), align: 'right' }, { value: worker.getLastUpdated(), align: 'left' }, { - value: - worker.status === 'OK' - ? `${worker.channelsPath}
${worker.guidePath}` - : '' + value: sources.length + ? sources + .map((source: WorkerGuideSource) => `${source.format}`) + .join(' | ') + : '-' } ]) ) }) logger.info('updating guides.md...') + const rootStorage = new Storage(ROOT_DIR) const table = new HTMLTable( rows, new Collection([ diff --git a/scripts/commands/workers/load.ts b/scripts/commands/workers/load.ts new file mode 100644 index 000000000..7145c2c2c --- /dev/null +++ b/scripts/commands/workers/load.ts @@ -0,0 +1,122 @@ +import epgGrabber, { EPGGrabber } from 'epg-grabber' +import { DATA_DIR, ROOT_DIR } from '../../constants' +import AxiosMockAdapter from 'axios-mock-adapter' +import { Storage } from '@freearhey/storage-js' +import { Channel, Worker } from '../../models' +import { Collection } from '@freearhey/core' +import { Logger } from '@freearhey/core' +import epgParser from 'epg-parser' +import axios from 'axios' +import path from 'path' + +async function main() { + const logger = new Logger({ level: process.env.NODE_ENV === 'test' ? -999 : 3 }) + const rootStorage = new Storage(ROOT_DIR) + const workers = new Map() + + logger.info('loading workers.txt...') + const workersTxt = await rootStorage.load('workers.txt') + const hosts = workersTxt.split('\r\n') + + hosts.forEach((host: string) => { + if (!host) return + + const worker = new Worker({ host }) + + workers.set(host, worker) + }) + + for (const worker of workers.values()) { + logger.info(`processing "${worker.host}"...`) + + const client = axios.create({ + baseURL: worker.getBaseUrl(), + timeout: 60000 + }) + + if (process.env.NODE_ENV === 'test') { + const mock = new AxiosMockAdapter(client) + if (worker.host === 'example.com') { + mock.onGet('worker.json').reply(404) + } else { + const testStorage = new Storage('tests/__data__/input/workers_load') + mock.onGet('worker.json').reply(200, await testStorage.load('worker.json')) + mock.onGet('channels.xml').reply(200, await testStorage.load('channels.xml')) + mock.onGet('guide.xml').reply(200, await testStorage.load('guide.xml')) + } + } + + const workerConfig = await client + .get('worker.json') + .then(res => res.data) + .catch(err => { + worker.setStatus(err.status) + logger.error(err.message) + }) + + if (!workerConfig) { + worker.setStatus('MISSING_WORKER_CONFIG') + logger.error('Unable to load "workers.json"') + continue + } + + worker + .setChannelsPath(workerConfig.channels) + .setGuideXmlPath( + typeof workerConfig.guide === 'string' ? workerConfig.guide : workerConfig?.guide?.xml + ) + .setGuideGzipPath(workerConfig?.guide?.gzip) + .setGuideJsonPath(workerConfig?.guide?.json) + + if (!worker.channelsPath) { + worker.setStatus('MISSING_CHANNELS_PATH') + logger.error('The "channels" property is missing from the workers config') + continue + } + + if (!worker.guideXmlPath) { + worker.setStatus('MISSING_GUIDE_XML_PATH') + logger.error('The "guide" property is missing from the workers config') + continue + } + + const channelsXml = await client + .get(worker.channelsPath) + .then(res => res.data) + .catch(err => { + worker.status = err.status + logger.error(err.message) + }) + + if (!channelsXml) continue + + const parsedChannels = EPGGrabber.parseChannelsXML(channelsXml) + worker.channels = new Collection(parsedChannels).map( + (channel: epgGrabber.Channel) => new Channel(channel.toObject()) + ) + + const guideXml = await client + .get(worker.guideXmlPath) + .then(res => res.data) + .catch(err => { + worker.setStatus(err.status) + logger.error(err.message) + }) + + if (!guideXml) continue + + const parsedGuide = epgParser.parse(guideXml) + worker.lastUpdated = parsedGuide.date + + worker.setStatus('OK') + } + + const output = [...workers.values()] + const dataStorage = new Storage(DATA_DIR) + const outputFilename = 'workers.json' + await dataStorage.save(outputFilename, JSON.stringify(output)) + + logger.info(`saved to "${path.join(DATA_DIR, outputFilename)}"`) +} + +main() diff --git a/scripts/default.config.js b/scripts/default.config.js index d5e49a641..056695ba7 100644 --- a/scripts/default.config.js +++ b/scripts/default.config.js @@ -5,6 +5,7 @@ export default { maxConnections: 1, curl: false, gzip: false, + json: false, debug: false, request: { maxContentLength: 5242880, diff --git a/scripts/models/apiGuide.ts b/scripts/models/apiGuide.ts new file mode 100644 index 000000000..bb558897a --- /dev/null +++ b/scripts/models/apiGuide.ts @@ -0,0 +1,49 @@ +import { WorkerGuideSource } from './worker' + +export interface ApiGuideData { + channel: string | null + feed: string | null + site: string + site_id: string + site_name: string + lang: string | null + sources?: WorkerGuideSource[] +} + +export class ApiGuide { + channel: string | null + feed: string | null + site: string + site_id: string + site_name: string + lang: string + sources: WorkerGuideSource[] = [] + + constructor(data: ApiGuideData) { + this.channel = data.channel || null + this.feed = data.feed || null + this.site = data.site + this.site_id = data.site_id + this.site_name = data.site_name + this.lang = data.lang || 'en' + this.sources = data.sources || [] + } + + addSource(source: ApiGuideSource): this { + this.sources.push(source) + + return this + } + + toJSON(): ApiGuideData { + return { + channel: this.channel, + feed: this.feed, + site: this.site, + site_id: this.site_id, + site_name: this.site_name, + lang: this.lang, + sources: this.sources + } + } +} diff --git a/scripts/models/channel.ts b/scripts/models/channel.ts index 087a39778..0ceef155b 100644 --- a/scripts/models/channel.ts +++ b/scripts/models/channel.ts @@ -1,22 +1,8 @@ -import { ChannelGuideObject } from '../types/channel' import * as epgGrabber from 'epg-grabber' import { SITES_DIR } from '../constants' import path from 'node:path' export class Channel extends epgGrabber.Channel { - getGuideObject(): ChannelGuideObject { - const [channelId, feedId] = this.xmltv_id.split('@') - - return { - channel: channelId || null, - feed: feedId || null, - site: this.site, - site_id: this.site_id, - site_name: this.name, - lang: this.lang || 'en' - } - } - getConfigPath(): string { return path.resolve(SITES_DIR, `${this.site}/${this.site}.config.js`) } diff --git a/scripts/models/guide.ts b/scripts/models/guide.ts index 8c367e6eb..cb68ae1f1 100644 --- a/scripts/models/guide.ts +++ b/scripts/models/guide.ts @@ -13,20 +13,23 @@ interface GuideData { channels: Collection programs: Collection filepath: string - gzip: boolean + gzip: boolean | string + json: boolean | string } export class Guide { channels: Collection programs: Collection filepath: string - gzip: boolean + gzip: boolean | string + json: boolean | string constructor(data: GuideData) { this.channels = data.channels this.programs = data.programs this.filepath = data.filepath this.gzip = data.gzip || false + this.json = data.json || false } addChannel(channel: Channel) { @@ -35,8 +38,9 @@ export class Guide { toString() { const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString()) + const headers = { date: currDate.format('YYYYMMDD') } - return EPGGrabber.generateXMLTV(this.channels.all(), this.programs.all(), currDate) + return EPGGrabber.generateXMLTV(this.channels.all(), this.programs.all(), headers) } async save({ logger }: { logger: Logger }) { @@ -50,10 +54,23 @@ export class Guide { if (this.gzip) { const compressed = pako.gzip(xmltv) - const gzFilepath = `${this.filepath}.gz` + const gzFilepath = typeof this.gzip === 'string' ? this.gzip : `${this.filepath}.gz` const gzFilename = path.basename(gzFilepath) logger.info(` saving to "${gzFilepath}"...`) await storage.save(gzFilename, compressed) } + + if (this.json) { + const filename = path.basename(this.filepath).split('.')[0] + const jsonFilepath = + typeof this.json === 'string' ? this.json : path.join(dir, `${filename}.json`) + const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString()) + const headers = { date: currDate.format('YYYYMMDD') } + const channels = this.channels.map((channel: Channel) => channel.toObject()).all() + const programs = this.programs.map((program: Program) => program.toObject()).all() + const json = JSON.stringify({ ...headers, channels, programs }) + logger.info(` saving to "${jsonFilepath}"...`) + await storage.save(jsonFilepath, json) + } } } diff --git a/scripts/models/worker.ts b/scripts/models/worker.ts index 94b94dfcb..f9e6f835b 100644 --- a/scripts/models/worker.ts +++ b/scripts/models/worker.ts @@ -1,5 +1,6 @@ import relativeTime from 'dayjs/plugin/relativeTime' import { Collection } from '@freearhey/core' +import * as epgGrabber from 'epg-grabber' import { Channel } from './channel' import utc from 'dayjs/plugin/utc' import dayjs from 'dayjs' @@ -7,20 +8,79 @@ import dayjs from 'dayjs' dayjs.extend(relativeTime) dayjs.extend(utc) +export interface WorkerGuideSource { + host: string + format: string + url: string +} + export interface WorkerData { host: string + channelsPath?: string + channels?: epgGrabber.Types.ChannelData[] + guideXmlPath?: string + guideGzipPath?: string + guideJsonPath?: string + status?: string + lastUpdated?: string } export class Worker { host: string channelsPath?: string - guidePath?: string channels?: Collection + guideXmlPath?: string + guideGzipPath?: string + guideJsonPath?: string status?: string lastUpdated?: string constructor(data: WorkerData) { this.host = data.host + + this.update(data) + } + + update(data: Partial): this { + if (data.host) this.host = data.host + if (data.channelsPath) this.channelsPath = data.channelsPath + if (data.guideXmlPath) this.guideXmlPath = data.guideXmlPath + if (data.guideGzipPath) this.guideGzipPath = data.guideGzipPath + if (data.guideJsonPath) this.guideJsonPath = data.guideJsonPath + if (data.status) this.status = data.status + if (data.lastUpdated) this.lastUpdated = data.lastUpdated + + if (data.channels) { + const channelInstances = data.channels.map(c => new Channel(c)) + this.channels = new Collection(channelInstances) + } + + return this + } + + setChannelsPath(path: string): this { + this.channelsPath = path + return this + } + + setGuideXmlPath(path: string): this { + this.guideXmlPath = path + return this + } + + setGuideGzipPath(path: string): this { + this.guideGzipPath = path + return this + } + + setGuideJsonPath(path: string): this { + this.guideJsonPath = path + return this + } + + setStatus(status: string): this { + this.status = status + return this } getBaseUrl(): string { @@ -41,10 +101,26 @@ export class Worker { return url.href } - getGuideUrl(): string { - if (!this.guidePath) return '' + getGuideXmlUrl(): string { + if (!this.guideXmlPath) return '' - const url = new URL(this.guidePath, this.getBaseUrl()) + const url = new URL(this.guideXmlPath, this.getBaseUrl()) + + return url.href + } + + getGuideGzipUrl(): string { + if (!this.guideGzipPath) return '' + + const url = new URL(this.guideGzipPath, this.getBaseUrl()) + + return url.href + } + + getGuideJsonUrl(): string { + if (!this.guideJsonPath) return '' + + const url = new URL(this.guideJsonPath, this.getBaseUrl()) return url.href } @@ -65,9 +141,42 @@ export class Worker { getLastUpdated(): string { if (!this.lastUpdated) return '-' - let now = dayjs() - if (process.env.NODE_ENV === 'test') now = dayjs.utc('2026-02-13') + const now = dayjs.utc(process.env.CURR_DATE || new Date().toISOString()) return dayjs.utc(this.lastUpdated).from(now) } + + getGuideSources(): WorkerGuideSource[] { + const sources = [] + + if (this.guideXmlPath) { + const url = new URL(this.guideXmlPath, this.getBaseUrl()) + sources.push({ host: this.host, url: url.href, format: 'XML' }) + } + + if (this.guideGzipPath) { + const url = new URL(this.guideGzipPath, this.getBaseUrl()) + sources.push({ host: this.host, url: url.href, format: 'GZIP' }) + } + + if (this.guideJsonPath) { + const url = new URL(this.guideJsonPath, this.getBaseUrl()) + sources.push({ host: this.host, url: url.href, format: 'JSON' }) + } + + return sources + } + + toJSON(): WorkerData { + return { + host: this.host, + channelsPath: this.channelsPath, + channels: this.channels ? this.channels.map(c => c.toObject()).all() : [], + guideXmlPath: this.guideXmlPath, + guideGzipPath: this.guideGzipPath, + guideJsonPath: this.guideJsonPath, + status: this.status, + lastUpdated: this.lastUpdated + } + } } diff --git a/scripts/types/channel.d.ts b/scripts/types/channel.d.ts deleted file mode 100644 index 3bd35d6fc..000000000 --- a/scripts/types/channel.d.ts +++ /dev/null @@ -1,8 +0,0 @@ -export interface ChannelGuideObject { - channel: string | null - feed: string | null - site: string - site_id: string - site_name: string - lang: string -} diff --git a/tests/__data__/expected/api_generate/guides.json b/tests/__data__/expected/api_generate/guides.json deleted file mode 100644 index b658a7821..000000000 --- a/tests/__data__/expected/api_generate/guides.json +++ /dev/null @@ -1 +0,0 @@ -[{"channel":"Channel1.us","feed":"SD","site":"example.com","site_id":"140","site_name":"Channel 1","lang":"en"},{"channel":"Channel2.us","feed":null,"site":"example.com","site_id":"142","site_name":"Channel 2","lang":"en"},{"channel":"Channel1.us","feed":"HD","site":"example.com","site_id":"140","site_name":"Channel 1","lang":"fr"},{"channel":"Channel3.us","feed":"HD","site":"example2.com","site_id":"150","site_name":"Channel 3","lang":"en"},{"channel":"Channel4.us","feed":null,"site":"example2.com","site_id":"152","site_name":"Channel 4","lang":"en"},{"channel":"Channel1.us","feed":null,"site":"example2.com","site_id":"140","site_name":"Channel 1","lang":"fr"}] \ No newline at end of file diff --git a/tests/__data__/expected/epg_grab/base.guide.xml b/tests/__data__/expected/epg_grab/base.guide.xml index a0ed3732f..16b14442b 100644 --- a/tests/__data__/expected/epg_grab/base.guide.xml +++ b/tests/__data__/expected/epg_grab/base.guide.xml @@ -1,8 +1,8 @@ -Channel 1https://example.com -Channel 2https://example.com36 -Channel 1https://example.com -Channel 3https://example.com +Channel 1 +Channel 236 +Channel 1 +Channel 3 Programme1 (example.com) Programme1 (example.com) Program1 (example.com) diff --git a/tests/__data__/expected/epg_grab/custom_channels.guide.xml b/tests/__data__/expected/epg_grab/custom_channels.guide.xml index 94215f489..b097cf402 100644 --- a/tests/__data__/expected/epg_grab/custom_channels.guide.xml +++ b/tests/__data__/expected/epg_grab/custom_channels.guide.xml @@ -1,10 +1,10 @@ -Custom Channel 1https://example.com -Channel 1https://example2.com -Channel 4https://example2.com -Channel 3https://example2.com -Channel 1https://example.com -Custom Channel 2https://example.com +Custom Channel 1 +Channel 1 +Channel 4 +Channel 3 +Channel 1 +Custom Channel 2 Program1 (example.com) Program1 (example.com) Programme1 (example2.com) diff --git a/tests/__data__/expected/epg_grab/gzip.guide.xml.gz b/tests/__data__/expected/epg_grab/gzip.guide.xml.gz deleted file mode 100644 index ccce7906c..000000000 Binary files a/tests/__data__/expected/epg_grab/gzip.guide.xml.gz and /dev/null differ diff --git a/tests/__data__/expected/epg_grab/gzip.guide.xml b/tests/__data__/expected/epg_grab/gzip/guide.xml similarity index 76% rename from tests/__data__/expected/epg_grab/gzip.guide.xml rename to tests/__data__/expected/epg_grab/gzip/guide.xml index 67968c1bd..bba9e9a18 100644 --- a/tests/__data__/expected/epg_grab/gzip.guide.xml +++ b/tests/__data__/expected/epg_grab/gzip/guide.xml @@ -1,7 +1,7 @@ -Channel 3https://example2.com -Channel 4https://example2.com -Channel 1https://example2.com +Channel 3 +Channel 4 +Channel 1 Programme1 (example2.com) Program1 (example2.com) Program1 (example2.com) diff --git a/tests/__data__/expected/epg_grab/gzip/guide.xml.gz b/tests/__data__/expected/epg_grab/gzip/guide.xml.gz new file mode 100644 index 000000000..3f0eaf110 Binary files /dev/null and b/tests/__data__/expected/epg_grab/gzip/guide.xml.gz differ diff --git a/tests/__data__/expected/epg_grab/json/guide.json b/tests/__data__/expected/epg_grab/json/guide.json new file mode 100644 index 000000000..83d442b77 --- /dev/null +++ b/tests/__data__/expected/epg_grab/json/guide.json @@ -0,0 +1 @@ +{"date":"20221020","channels":[{"xmltv_id":"Channel3.us","name":"Channel 3","site":"example2.com","site_id":"150","lang":"en","logo":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png","url":null,"lcn":null,"index":0},{"xmltv_id":"Channel4.us@HD","name":"Channel 4","site":"example2.com","site_id":"152","lang":"en","logo":"https://i.imgur.com/BPzH88J.png","url":null,"lcn":null,"index":1},{"xmltv_id":"Channel1.us","name":"Channel 1","site":"example2.com","site_id":"140","lang":"fr","logo":"https://i.imgur.com/GPzH88J.png","url":null,"lcn":null,"index":2}],"programs":[{"site":"example2.com","start":1666240800000,"stop":1666249800000,"channel":"Channel1.us","titles":[{"value":"Programme1 (example2.com)","lang":"fr"}],"subTitles":[],"descriptions":[],"date":0,"categories":[],"keywords":[],"languages":[],"origLanguages":[],"length":[],"urls":[],"countries":[],"episodeNumbers":[],"video":{"present":"","colour":"","aspect":"","quality":""},"audio":{"present":"","stereo":""},"previouslyShown":[],"premiere":[],"lastChance":[],"new":false,"subtitles":[],"ratings":[],"starRatings":[],"reviews":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"images":[],"icons":[]},{"site":"example2.com","start":1666240260000,"stop":1666249800000,"channel":"Channel3.us","titles":[{"value":"Program1 (example2.com)","lang":"en"}],"subTitles":[],"descriptions":[],"date":0,"categories":[],"keywords":[],"languages":[],"origLanguages":[],"length":[],"urls":[],"countries":[],"episodeNumbers":[],"video":{"present":"","colour":"","aspect":"","quality":""},"audio":{"present":"","stereo":""},"previouslyShown":[],"premiere":[],"lastChance":[],"new":false,"subtitles":[],"ratings":[],"starRatings":[],"reviews":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"images":[],"icons":[]},{"site":"example2.com","start":1666240260000,"stop":1666249800000,"channel":"Channel4.us@HD","titles":[{"value":"Program1 (example2.com)","lang":"en"}],"subTitles":[],"descriptions":[],"date":0,"categories":[],"keywords":[],"languages":[],"origLanguages":[],"length":[],"urls":[],"countries":[],"episodeNumbers":[],"video":{"present":"","colour":"","aspect":"","quality":""},"audio":{"present":"","stereo":""},"previouslyShown":[],"premiere":[],"lastChance":[],"new":false,"subtitles":[],"ratings":[],"starRatings":[],"reviews":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"images":[],"icons":[]}]} \ No newline at end of file diff --git a/tests/__data__/expected/epg_grab/json/guide.xml b/tests/__data__/expected/epg_grab/json/guide.xml new file mode 100644 index 000000000..bba9e9a18 --- /dev/null +++ b/tests/__data__/expected/epg_grab/json/guide.xml @@ -0,0 +1,8 @@ + +Channel 3 +Channel 4 +Channel 1 +Programme1 (example2.com) +Program1 (example2.com) +Program1 (example2.com) + \ No newline at end of file diff --git a/tests/__data__/expected/epg_grab/lang/fr/guide.xml b/tests/__data__/expected/epg_grab/lang/fr/guide.xml index dde894e5f..15f36ee2f 100644 --- a/tests/__data__/expected/epg_grab/lang/fr/guide.xml +++ b/tests/__data__/expected/epg_grab/lang/fr/guide.xml @@ -1,5 +1,5 @@ -Channel 1https://example.com +Channel 1 Programme1 (example.com) Programme1 (example.com) \ No newline at end of file diff --git a/tests/__data__/expected/epg_grab/multilang.guide.xml b/tests/__data__/expected/epg_grab/multilang.guide.xml index 40e5b5aa6..bd0209bd5 100644 --- a/tests/__data__/expected/epg_grab/multilang.guide.xml +++ b/tests/__data__/expected/epg_grab/multilang.guide.xml @@ -1,6 +1,6 @@ -Channel 1https://example.com -Channel 3https://example.com +Channel 1 +Channel 3 Programme1 (example.com) Programme1 (example.com) Program1 (example.com) diff --git a/tests/__data__/expected/epg_grab/multiple_channels.guide.xml b/tests/__data__/expected/epg_grab/multiple_channels.guide.xml index 1a2ececa4..b2b76d31a 100644 --- a/tests/__data__/expected/epg_grab/multiple_channels.guide.xml +++ b/tests/__data__/expected/epg_grab/multiple_channels.guide.xml @@ -1,11 +1,11 @@ -Channel 1https://example.com -Channel 1https://example2.com -Channel 4https://example2.com -Channel 3https://example2.com -Channel 3https://example.com -Channel 1https://example.com -Channel 2https://example.com36 +Channel 1 +Channel 1 +Channel 4 +Channel 3 +Channel 3 +Channel 1 +Channel 236 Program1 (example.com) Program1 (example.com) Programme1 (example2.com) diff --git a/tests/__data__/expected/epg_grab/wildcard/example.com/en/guide.xml b/tests/__data__/expected/epg_grab/wildcard/example.com/en/guide.xml index 6426626a0..ed1cb2d20 100644 --- a/tests/__data__/expected/epg_grab/wildcard/example.com/en/guide.xml +++ b/tests/__data__/expected/epg_grab/wildcard/example.com/en/guide.xml @@ -1,6 +1,6 @@ -Channel 1https://example.com -Channel 2https://example.com36 +Channel 1 +Channel 236 Program1 (example.com) Program1 (example.com) \ No newline at end of file diff --git a/tests/__data__/expected/epg_grab/wildcard/example.com/fr/guide.xml b/tests/__data__/expected/epg_grab/wildcard/example.com/fr/guide.xml index dde894e5f..15f36ee2f 100644 --- a/tests/__data__/expected/epg_grab/wildcard/example.com/fr/guide.xml +++ b/tests/__data__/expected/epg_grab/wildcard/example.com/fr/guide.xml @@ -1,5 +1,5 @@ -Channel 1https://example.com +Channel 1 Programme1 (example.com) Programme1 (example.com) \ No newline at end of file diff --git a/tests/__data__/expected/epg_grab/wildcard/example.com/it/guide.xml b/tests/__data__/expected/epg_grab/wildcard/example.com/it/guide.xml index 9d839e5e3..37cfd2ed3 100644 --- a/tests/__data__/expected/epg_grab/wildcard/example.com/it/guide.xml +++ b/tests/__data__/expected/epg_grab/wildcard/example.com/it/guide.xml @@ -1,5 +1,5 @@ -Channel 3https://example.com +Channel 3 Program1 (example.com) Program1 (example.com) \ No newline at end of file diff --git a/tests/__data__/expected/guides_export/guides.json b/tests/__data__/expected/guides_export/guides.json new file mode 100644 index 000000000..1239c7edf --- /dev/null +++ b/tests/__data__/expected/guides_export/guides.json @@ -0,0 +1 @@ +[{"channel":"Channel1.us","feed":"SD","site":"example.com","site_id":"140","site_name":"Channel 1","lang":"en","sources":[]},{"channel":"Channel2.us","feed":null,"site":"example.com","site_id":"142","site_name":"Channel 2","lang":"en","sources":[]},{"channel":"Channel1.us","feed":"HD","site":"example.com","site_id":"140","site_name":"Channel 1","lang":"fr","sources":[]},{"channel":"Channel3.us","feed":"HD","site":"example2.com","site_id":"150","site_name":"Channel 3","lang":"en","sources":[]},{"channel":"Channel4.us","feed":null,"site":"example2.com","site_id":"152","site_name":"Channel 4","lang":"en","sources":[]},{"channel":"Channel1.us","feed":null,"site":"example2.com","site_id":"140","site_name":"Channel 1","lang":"fr","sources":[]}] \ No newline at end of file diff --git a/tests/__data__/expected/guides_update/GUIDES.md b/tests/__data__/expected/guides_update/GUIDES.md index f6e86cfc8..2abafcce9 100644 --- a/tests/__data__/expected/guides_update/GUIDES.md +++ b/tests/__data__/expected/guides_update/GUIDES.md @@ -5,8 +5,8 @@ HostStatusChannelsLast UpdatedLinks - example.com🔴0- - worker-9dd4.onrender.com🟢1a day agochannels.xml
guide.xml + example.com🔴0-- + worker-9dd4.onrender.com🟢1a day agoXML | GZIP | JSON diff --git a/tests/__data__/expected/workers_load/workers.json b/tests/__data__/expected/workers_load/workers.json new file mode 100644 index 000000000..5884dd7af --- /dev/null +++ b/tests/__data__/expected/workers_load/workers.json @@ -0,0 +1 @@ +[{"host":"example.com","channels":[],"status":"MISSING_WORKER_CONFIG"},{"host":"worker-9dd4.onrender.com","channelsPath":"channels.xml","channels":[{"xmltv_id":"ANT1Europe.gr@SD","name":"ANT1 Europe","site":"antennaeurope.gr","site_id":"#","lang":"el","logo":null,"url":null,"lcn":null,"index":0}],"guideXmlPath":"guide.xml","guideGzipPath":"guide.xml.gz","guideJsonPath":"guide.json","status":"OK","lastUpdated":"2022-10-03T00:00:00.000Z"}] \ No newline at end of file diff --git a/tests/__data__/input/api_generate/sites/example.com/example.com.channels.xml b/tests/__data__/input/guides_export/sites/example.com/example.com.channels.xml similarity index 98% rename from tests/__data__/input/api_generate/sites/example.com/example.com.channels.xml rename to tests/__data__/input/guides_export/sites/example.com/example.com.channels.xml index 630e9225e..c0c4c2ffa 100644 --- a/tests/__data__/input/api_generate/sites/example.com/example.com.channels.xml +++ b/tests/__data__/input/guides_export/sites/example.com/example.com.channels.xml @@ -1,6 +1,6 @@ - - - Channel 1 - Channel 2 - Channel 1 + + + Channel 1 + Channel 2 + Channel 1 \ No newline at end of file diff --git a/tests/__data__/input/api_generate/sites/example2.com/example2.com.channels.xml b/tests/__data__/input/guides_export/sites/example2.com/example2.com.channels.xml similarity index 98% rename from tests/__data__/input/api_generate/sites/example2.com/example2.com.channels.xml rename to tests/__data__/input/guides_export/sites/example2.com/example2.com.channels.xml index f95bdb8dc..edb959bae 100644 --- a/tests/__data__/input/api_generate/sites/example2.com/example2.com.channels.xml +++ b/tests/__data__/input/guides_export/sites/example2.com/example2.com.channels.xml @@ -1,6 +1,6 @@ - - - Channel 3 - Channel 4 - Channel 1 + + + Channel 3 + Channel 4 + Channel 1 \ No newline at end of file diff --git a/tests/__data__/input/guides_update/worker.json b/tests/__data__/input/guides_update/worker.json deleted file mode 100644 index 371feec07..000000000 --- a/tests/__data__/input/guides_update/worker.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "channels": "channels.xml", - "guide": "guide.xml" -} \ No newline at end of file diff --git a/tests/__data__/input/guides_update/workers.json b/tests/__data__/input/guides_update/workers.json new file mode 100644 index 000000000..314483d59 --- /dev/null +++ b/tests/__data__/input/guides_update/workers.json @@ -0,0 +1,29 @@ +[ + { + "host": "example.com", + "channels": [], + "status": "MISSING_WORKER_CONFIG" + }, + { + "host": "worker-9dd4.onrender.com", + "channelsPath": "channels.xml", + "channels": [ + { + "xmltv_id": "ANT1Europe.gr@SD", + "name": "ANT1 Europe", + "site": "antennaeurope.gr", + "site_id": "#", + "lang": "el", + "logo": null, + "url": null, + "lcn": null, + "index": 0 + } + ], + "guideXmlPath": "guide.xml", + "guideGzipPath": "guide.xml.gz", + "guideJsonPath": "guide.json", + "status": "OK", + "lastUpdated": "2022-10-03T00:00:00.000Z" + } +] \ No newline at end of file diff --git a/tests/__data__/input/guides_update/channels.xml b/tests/__data__/input/workers_load/channels.xml similarity index 100% rename from tests/__data__/input/guides_update/channels.xml rename to tests/__data__/input/workers_load/channels.xml diff --git a/tests/__data__/input/guides_update/guide.xml b/tests/__data__/input/workers_load/guide.xml similarity index 97% rename from tests/__data__/input/guides_update/guide.xml rename to tests/__data__/input/workers_load/guide.xml index ed17438ed..ddc29c7b0 100644 --- a/tests/__data__/input/guides_update/guide.xml +++ b/tests/__data__/input/workers_load/guide.xml @@ -1,4 +1,4 @@ - + ANT1 Europehttps://antennaeurope.gr ΚΑΛΗΜΕΡΑ ΕΛΛΑΔΑ ΤΟ ΠΡΩΙΝΟ diff --git a/tests/__data__/input/workers_load/worker.json b/tests/__data__/input/workers_load/worker.json new file mode 100644 index 000000000..35ceb014e --- /dev/null +++ b/tests/__data__/input/workers_load/worker.json @@ -0,0 +1,8 @@ +{ + "channels": "channels.xml", + "guide": { + "xml": "guide.xml", + "gzip": "guide.xml.gz", + "json": "guide.json" + } +} \ No newline at end of file diff --git a/tests/__data__/input/guides_update/workers.txt b/tests/__data__/input/workers_load/workers.txt similarity index 100% rename from tests/__data__/input/guides_update/workers.txt rename to tests/__data__/input/workers_load/workers.txt diff --git a/tests/commands/epg/grab.test.ts b/tests/commands/epg/grab.test.ts index b2a965c3b..743cb0fb7 100644 --- a/tests/commands/epg/grab.test.ts +++ b/tests/commands/epg/grab.test.ts @@ -90,21 +90,20 @@ describe('epg:grab', () => { it('can grab epg with gzip option enabled', () => { const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example2.com/example2.com.channels.xml --output="${path.resolve( - 'tests/__data__/output/guides/gzip.guide.xml' - )}" --gzip ` + 'tests/__data__/output/guides/guide.xml' + )}" --gzip` const stdout = execSync(cmd, { encoding: 'utf8' }) if (process.env.DEBUG === 'true') console.log(cmd, stdout) - expect(content('tests/__data__/output/guides/gzip.guide.xml')).toEqual( - content('tests/__data__/expected/epg_grab/gzip.guide.xml') + expect(content('tests/__data__/output/guides/guide.xml')).toEqual( + content('tests/__data__/expected/epg_grab/gzip/guide.xml') ) - const outputString = pako.ungzip( - fs.readFileSync('tests/__data__/output/guides/gzip.guide.xml.gz'), - { to: 'string' } - ) + const outputString = pako.ungzip(fs.readFileSync('tests/__data__/output/guides/guide.xml.gz'), { + to: 'string' + }) const expectedString = pako.ungzip( - fs.readFileSync('tests/__data__/expected/epg_grab/gzip.guide.xml.gz'), + fs.readFileSync('tests/__data__/expected/epg_grab/gzip/guide.xml.gz'), { to: 'string' } ) @@ -113,6 +112,66 @@ describe('epg:grab', () => { expect(output).toEqual(expected) }) + + it('can grab epg with gzip option enabled', () => { + const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example2.com/example2.com.channels.xml --output="${path.resolve( + 'tests/__data__/output/guides/guide.xml' + )}" --gzip="${path.resolve('tests/__data__/output/guides/custom.xml.gz')}"` + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + + expect(content('tests/__data__/output/guides/guide.xml')).toEqual( + content('tests/__data__/expected/epg_grab/gzip/guide.xml') + ) + + const outputString = pako.ungzip( + fs.readFileSync('tests/__data__/output/guides/custom.xml.gz'), + { + to: 'string' + } + ) + const expectedString = pako.ungzip( + fs.readFileSync('tests/__data__/expected/epg_grab/gzip/guide.xml.gz'), + { to: 'string' } + ) + + const output = new Set(outputString.split('\r\n')) + const expected = new Set(expectedString.split('\r\n')) + + expect(output).toEqual(expected) + }) + + it('can grab epg with json option enabled', () => { + const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example2.com/example2.com.channels.xml --output="${path.resolve( + 'tests/__data__/output/guides/guide.xml' + )}" --json` + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + + expect(content('tests/__data__/output/guides/guide.xml')).toEqual( + content('tests/__data__/expected/epg_grab/json/guide.xml') + ) + + expect(content('tests/__data__/output/guides/guide.json')).toEqual( + content('tests/__data__/expected/epg_grab/json/guide.json') + ) + }) + + it('can grab epg with json option enabled', () => { + const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example2.com/example2.com.channels.xml --output="${path.resolve( + 'tests/__data__/output/guides/guide.xml' + )}" --json="${path.resolve('tests/__data__/output/guides/custom.json')}"` + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + + expect(content('tests/__data__/output/guides/guide.xml')).toEqual( + content('tests/__data__/expected/epg_grab/json/guide.xml') + ) + + expect(content('tests/__data__/output/guides/custom.json')).toEqual( + content('tests/__data__/expected/epg_grab/json/guide.json') + ) + }) }) function content(filepath: string) { diff --git a/tests/commands/api/generate.test.ts b/tests/commands/guides/export.test.ts similarity index 64% rename from tests/commands/api/generate.test.ts rename to tests/commands/guides/export.test.ts index 2044f0d9c..ee787e8a2 100644 --- a/tests/commands/api/generate.test.ts +++ b/tests/commands/guides/export.test.ts @@ -1,27 +1,28 @@ -import { execSync } from 'child_process' -import fs from 'fs-extra' -import { pathToFileURL } from 'node:url' - -const ENV_VAR = 'cross-env SITES_DIR=tests/__data__/input/api_generate/sites API_DIR=tests/__data__/output' - -beforeEach(() => { - fs.emptyDirSync('tests/__data__/output') -}) - -describe('api:generate', () => { - it('can generate guides.json', () => { - const cmd = `${ENV_VAR} npm run api:generate` - const stdout = execSync(cmd, { encoding: 'utf8' }) - if (process.env.DEBUG === 'true') console.log(cmd, stdout) - - expect(content('tests/__data__/output/guides.json')).toEqual( - content('tests/__data__/expected/api_generate/guides.json') - ) - }) -}) - -function content(filepath: string) { - return fs.readFileSync(pathToFileURL(filepath), { - encoding: 'utf8' - }) -} +import { execSync } from 'child_process' +import fs from 'fs-extra' +import { pathToFileURL } from 'node:url' + +const ENV_VAR = + 'cross-env SITES_DIR=tests/__data__/input/guides_export/sites API_DIR=tests/__data__/output' + +beforeEach(() => { + fs.emptyDirSync('tests/__data__/output') +}) + +describe('guides:export', () => { + it('can generate guides.json', () => { + const cmd = `${ENV_VAR} npm run guides:export` + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + + expect(content('tests/__data__/output/guides.json')).toEqual( + content('tests/__data__/expected/guides_export/guides.json') + ) + }) +}) + +function content(filepath: string) { + return fs.readFileSync(pathToFileURL(filepath), { + encoding: 'utf8' + }) +} diff --git a/tests/commands/guides/update.test.ts b/tests/commands/guides/update.test.ts index 4f3d0f6ef..0a4da2639 100644 --- a/tests/commands/guides/update.test.ts +++ b/tests/commands/guides/update.test.ts @@ -2,22 +2,24 @@ import { execSync } from 'child_process' import { pathToFileURL } from 'node:url' import fs from 'fs-extra' -const ENV_VAR = 'cross-env ROOT_DIR=tests/__data__/output' +const ENV_VAR = + 'cross-env CURR_DATE=2022-10-04 ROOT_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/guides_update' beforeEach(() => { fs.emptyDirSync('tests/__data__/output') - fs.copySync('tests/__data__/input/guides_update/workers.txt', 'tests/__data__/output/workers.txt') }) -it('can update GUIDES.md', () => { - const cmd = `${ENV_VAR} npm run guides:update` +describe('guides:update', () => { + it('can update GUIDES.md', () => { + const cmd = `${ENV_VAR} npm run guides:update` - const stdout = execSync(cmd, { encoding: 'utf8' }) - if (process.env.DEBUG === 'true') console.log(cmd, stdout) + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) - expect(content('tests/__data__/output/GUIDES.md')).toEqual( - content('tests/__data__/expected/guides_update/GUIDES.md') - ) + expect(content('tests/__data__/output/GUIDES.md')).toEqual( + content('tests/__data__/expected/guides_update/GUIDES.md') + ) + }) }) function content(filepath: string) { diff --git a/tests/commands/workers/load.test.ts b/tests/commands/workers/load.test.ts new file mode 100644 index 000000000..724e0f50e --- /dev/null +++ b/tests/commands/workers/load.test.ts @@ -0,0 +1,32 @@ +import { execSync } from 'child_process' +import { pathToFileURL } from 'node:url' +import fs from 'fs-extra' + +const ENV_VAR = + 'cross-env CURR_DATE=2022-10-04 ROOT_DIR=tests/__data__/output DATA_DIR=tests/__data__/output/' + +beforeEach(() => { + fs.emptyDirSync('tests/__data__/output') + fs.copySync('tests/__data__/input/workers_load/workers.txt', 'tests/__data__/output/workers.txt') +}) + +describe('workers:load', () => { + it('can create workers.json', () => { + const cmd = `${ENV_VAR} npm run workers:load` + + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + + expect(content('tests/__data__/output/workers.json')).toEqual( + content('tests/__data__/expected/workers_load/workers.json') + ) + }) +}) + +function content(filepath: string) { + const data = fs.readFileSync(pathToFileURL(filepath), { + encoding: 'utf8' + }) + + return JSON.stringify(data) +} diff --git a/tsconfig.json b/tsconfig.json index 9e9476b15..4fc2a0295 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,6 +13,6 @@ "allowJs": true, "outDir": "./dist" }, - "include": ["scripts/**/*", "src/**/*"], + "include": ["scripts/**/*", "tests/**/*"], "exclude": ["node_modules", "dist"] } \ No newline at end of file