Merge pull request #3137 from iptv-org/patch-2026.05.2

Patch 2026.05.2

.github/workflows/update.yml

@@ -45,6 +45,8 @@ jobs:
             git commit -m "[Bot] Update ${FILE}" -m "Committed by [iptv-bot](https://github.com/apps/iptv-bot) via [update](https://github.com/iptv-org/epg/actions/runs/${{ github.run_id }}) workflow." --no-verify
             git push
           fi
+      - name: load workers
+        run: npm run workers:load
       - name: update guides.md
         run: npm run guides:update
       - run: git status
@@ -59,7 +61,7 @@ jobs:
             git push
           fi
       - name: generate .api/guides.json
-        run: npm run api:generate
+        run: npm run guides:export
       - run: git status
       - name: move .api/guides.json to iptv-org/api
         uses: JamesIves/github-pages-deploy-action@4.1.1

CONTRIBUTING.md

@@ -422,6 +422,19 @@ To do this, you just need to place the `worker.json` file in a public directory
 }
 ```
 
+If you generate a guide in multiple formats, you can specify the paths to all the files as follows:
+
+```json
+{
+  "channels": "path/to/channels.xml",
+  "guide": {
+    "xml": "path/to/guide.xml",
+    "gzip": "path/to/guide.xml.gz",
+    "json": "path/to/guide.json"
+  }
+}
+```
+
 </details>
 
 And then add server domain or IP address to the [workers.txt](workers.txt) file.
@@ -454,7 +467,6 @@ To run scripts use the `npm run <script-name>` command.
 - `act:check`: allows to test the [check](https://github.com/iptv-org/iptv/blob/master/.github/workflows/check.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act).
 - `act:update`: allows to test the [update](https://github.com/iptv-org/iptv/blob/master/.github/workflows/update.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act).
 - `api:load`: downloads the latest channels data from the [iptv-org/api](https://github.com/iptv-org/api).
-- `api:generate`: generates a JSON file with all channels for the [iptv-org/api](https://github.com/iptv-org/api) repository.
 - `channels:lint`: сhecks the channel lists for syntax errors.
 - `channels:parse`: generates a list of channels based on the site configuration.
 - `channels:format`: formats `*.channels.xml` files. The process involves removing invalid `xmltv_id`, adding missing Feed ID, and sorting the list.
@@ -463,6 +475,8 @@ To run scripts use the `npm run <script-name>` command.
 - `sites:init`: creates a new site config from the template.
 - `sites:update`: updates the list of sites and their status in [SITES.md](SITES.md).
 - `guides:update`: updates the list of guides and their status in [GUIDES.md](GUIDES.md).
+- `guides:export`: exports a list of all guides in JSON format for the [iptv-org/api](https://github.com/iptv-org/api) repository.
+- `workers:load`: loads a list of all workers and their configurations.
 - `grab`: downloads a program from a specified source.
 - `serve`: starts the [web server](https://github.com/vercel/serve).
 - `lint`: сhecks the scripts for syntax errors.

README.md

@@ -55,7 +55,7 @@ Usage: npm run grab --- [options]
 
 Options:
   -s, --sites <names>           A comma-separated list of the sites to parse
-  -c, --channels <path>         Path to *.channels.xml file (required if the "--site" attribute is
+  -c, --channels <path>         Path to *.channels.xml file (required if the "--sites" attribute is
                                 not specified)
   -o, --output <path>           Path to output file (default: "guide.xml")
   -l, --lang <codes>            Allows you to restrict downloading to channels in specified languages only (example: "en,id")
@@ -64,7 +64,8 @@ Options:
   -x, --proxy <url>             Use the specified proxy (example: "socks5://username:password@127.0.0.1:1234")
   --days <days>                 Number of days for which the program will be loaded (defaults to the value from the site config)
   --maxConnections <number>     Number of concurrent requests (default: 1)
-  --gzip                        Specifies whether or not to create a compressed version of the guide (default: false)
+  --gzip [path]                 Specifies whether or not to create a compressed version of the guide (default: false)
+  --json [path]                 Specifies whether or not to create a JSON version of the guide (default: false)
   --curl                        Display each request as CURL (default: false)
 ```
 
@@ -197,11 +198,13 @@ docker run \
 -e CRON_SCHEDULE="0 0,12 * * *" \
 -e MAX_CONNECTIONS=10 \
 -e GZIP=true \
+-e JSON=true \
 -e CURL=true \
 -e PROXY="socks5://127.0.0.1:1234" \
 -e DAYS=14 \
 -e TIMEOUT=5 \
 -e DELAY=2 \
+-e RUN_AT_STARTUP=true \
 ghcr.io/iptv-org/epg:master
 ```
 
@@ -210,6 +213,7 @@ ghcr.io/iptv-org/epg:master
 | CRON_SCHEDULE   | A [cron expression](https://crontab.guru/) describing the schedule of the guide loadings (default: "0 0 \* \* \*") |
 | MAX_CONNECTIONS | Limit on the number of concurrent requests (default: 1)                                                            |
 | GZIP            | Boolean value indicating whether to create a compressed version of the guide (default: false)                      |
+| JSON            | Boolean value indicating whether to create a JSON version of the guide (default: false)                            |
 | CURL            | Display each request as CURL (default: false)                                                                      |
 | PROXY           | Use the specified proxy                                                                                            |
 | DAYS            | Number of days for which the guide will be loaded (defaults to the value from the site config)                     |

package-lock.json

@@ -44,7 +44,7 @@
         "csv-parser": "^3.2.0",
         "curl-generator": "^0.5.0",
         "dayjs": "^1.11.20",
-        "epg-grabber": "^0.46.2",
+        "epg-grabber": "^0.47.0",
         "epg-parser": "^0.5.0",
         "eslint": "^10.2.1",
         "eslint-config-prettier": "^10.1.8",
@@ -5632,9 +5632,9 @@
       }
     },
     "node_modules/epg-grabber": {
-      "version": "0.46.2",
+      "version": "0.47.0",
-      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.46.2.tgz",
+      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.47.0.tgz",
-      "integrity": "sha512-pcZD9eo0d5QFqLr/YUXh4SapjrE3MfHvjyIokVpfCVI5wVXkdy3q9SxM8b34hPYPy2caeaCTpZM1yTAeoIOIMw==",
+      "integrity": "sha512-kl+erC90n1ipt2aThPF9o7bkxYsS/QIUQ56M/RWfOL9vyKZxSU9vvzMDp88HwCZ6NWMYPpeDTooA1XnsWT86Eg==",
       "license": "MIT",
       "dependencies": {
         "@freearhey/core": "^0.14.0",
@@ -14952,9 +14952,9 @@
       "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA=="
     },
     "epg-grabber": {
-      "version": "0.46.2",
+      "version": "0.47.0",
-      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.46.2.tgz",
+      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.47.0.tgz",
-      "integrity": "sha512-pcZD9eo0d5QFqLr/YUXh4SapjrE3MfHvjyIokVpfCVI5wVXkdy3q9SxM8b34hPYPy2caeaCTpZM1yTAeoIOIMw==",
+      "integrity": "sha512-kl+erC90n1ipt2aThPF9o7bkxYsS/QIUQ56M/RWfOL9vyKZxSU9vvzMDp88HwCZ6NWMYPpeDTooA1XnsWT86Eg==",
       "requires": {
         "@freearhey/core": "^0.14.0",
         "@types/bluebird": "^3.5.42",

package.json

@@ -5,7 +5,6 @@
     "act:format": "act workflow_dispatch -W .github/workflows/format.yml",
     "act:update": "act workflow_dispatch -W .github/workflows/update.yml",
     "api:load": "tsx scripts/commands/api/load.ts",
-    "api:generate": "tsx scripts/commands/api/generate.ts",
     "channels:lint": "tsx scripts/commands/channels/lint.mts",
     "channels:parse": "tsx scripts/commands/channels/parse.ts",
     "channels:format": "tsx scripts/commands/channels/format.ts",
@@ -14,6 +13,8 @@
     "sites:init": "tsx scripts/commands/sites/init.ts",
     "sites:update": "tsx scripts/commands/sites/update.ts",
     "guides:update": "tsx scripts/commands/guides/update.ts",
+    "guides:export": "tsx scripts/commands/guides/export.ts",
+    "workers:load": "tsx scripts/commands/workers/load.ts",
     "grab": "tsx scripts/commands/epg/grab.ts",
     "lint": "npx eslint \"{scripts,tests,sites}/**/*.{ts,mts,js}\"",
     "test": "cross-env TZ=Pacific/Nauru npx jest --runInBand",
@@ -72,7 +73,7 @@
     "csv-parser": "^3.2.0",
     "curl-generator": "^0.5.0",
     "dayjs": "^1.11.20",
-    "epg-grabber": "^0.46.2",
+    "epg-grabber": "^0.47.0",
     "epg-parser": "^0.5.0",
     "eslint": "^10.2.1",
     "eslint-config-prettier": "^10.1.8",

scripts/commands/api/generate.ts

@@ -1,43 +0,0 @@
-import { ChannelGuideObject } from '../../types/channel'
-import { SITES_DIR, API_DIR } from '../../constants'
-import { Logger, Collection } from '@freearhey/core'
-import epgGrabber, { EPGGrabber } from 'epg-grabber'
-import { Storage } from '@freearhey/storage-js'
-import { Channel } from '../../models'
-import path from 'path'
-
-async function main() {
-  const logger = new Logger()
-
-  logger.start('staring...')
-
-  logger.info('loading channels...')
-  const sitesStorage = new Storage(SITES_DIR)
-
-  const files: string[] = await sitesStorage.list('**/*.channels.xml')
-
-  const channels = new Collection<Channel>()
-  for (const filepath of files) {
-    const xml = await sitesStorage.load(filepath)
-    const parsedChannels = EPGGrabber.parseChannelsXML(xml)
-    const channelsFromXML = new Collection(parsedChannels).map(
-      (channel: epgGrabber.Channel) => new Channel(channel.toObject())
-    )
-
-    channelsFromXML.forEach((channel: Channel) => {
-      channels.add(channel)
-    })
-  }
-
-  logger.info(`found ${channels.count()} channel(s)`)
-
-  const output = channels.map<ChannelGuideObject>((channel: Channel) => channel.getGuideObject())
-
-  const apiStorage = new Storage(API_DIR)
-  const outputFilename = 'guides.json'
-  await apiStorage.save('guides.json', output.toJSON())
-
-  logger.info(`saved to "${path.join(API_DIR, outputFilename)}"`)
-}
-
-main()

scripts/commands/epg/grab.ts

@@ -54,7 +54,10 @@ program
       .argParser(parseNumber)
       .env('MAX_CONNECTIONS')
   )
-  .addOption(new Option('--gzip', 'Create a compressed version of the guide as well').env('GZIP'))
+  .addOption(
+    new Option('--gzip [path]', 'Create a compressed version of the guide as well').env('GZIP')
+  )
+  .addOption(new Option('--json [path]', 'Create a JSON version of the guide as well').env('JSON'))
   .addOption(new Option('--curl', 'Display each request as CURL').env('CURL'))
   .addOption(new Option('--debug', 'Enable debug mode').env('DEBUG'))
   .parse()
@@ -63,7 +66,8 @@ interface GrabOptions {
   sites?: string[]
   channels?: string
   output?: string
-  gzip?: boolean
+  gzip?: boolean | string
+  json?: boolean | string
   curl?: boolean
   debug?: boolean
   maxConnections?: number
@@ -109,7 +113,10 @@ async function main() {
   if (typeof options.maxConnections === 'number')
     globalConfig.maxConnections = options.maxConnections
   if (typeof options.curl === 'boolean') globalConfig.curl = options.curl
-  if (typeof options.gzip === 'boolean') globalConfig.gzip = options.gzip
+  if (typeof options.gzip === 'boolean' || typeof options.gzip === 'string')
+    globalConfig.gzip = options.gzip
+  if (typeof options.json === 'boolean' || typeof options.json === 'string')
+    globalConfig.json = options.json
   if (typeof options.debug === 'boolean') globalConfig.debug = options.debug
 
   logger.debug(`config: ${JSON.stringify(globalConfig, null, 2)}`)
@@ -298,6 +305,7 @@ async function main() {
     })
 
   const gzip = globalConfig.gzip || defaultConfig.gzip
+  const json = globalConfig.json || defaultConfig.json
 
   for (const groupKey of channelsGroupedByKey.keys()) {
     const groupChannels = new Collection(channelsGroupedByKey.get(groupKey))
@@ -305,6 +313,7 @@ async function main() {
     const guide = new Guide({
       filepath: groupKey,
       gzip,
+      json,
       channels: groupChannels,
       programs: groupPrograms
     })

scripts/commands/guides/export.ts

@@ -0,0 +1,88 @@
+import { SITES_DIR, API_DIR, DATA_DIR } from '../../constants'
+import { Logger, Collection } from '@freearhey/core'
+import epgGrabber, { EPGGrabber } from 'epg-grabber'
+import { Storage } from '@freearhey/storage-js'
+import { Channel, Worker, WorkerData } from '../../models'
+import path from 'path'
+import { ApiGuide } from '../../models/apiGuide'
+
+async function main() {
+  const logger = new Logger()
+
+  logger.start('staring...')
+
+  logger.info('loading channels...')
+  const channels = await loadChannels()
+  logger.info(`found ${channels.count()} channel(s)`)
+
+  logger.info('loading workers.json...')
+  const workers = await loadWorkers()
+  const guidesMap = workers.reduce((acc, data: WorkerData) => {
+    const worker = new Worker(data)
+    if (!worker.channels) return acc
+    worker.channels.forEach(channel => {
+      const [channelId, feedId] = channel.xmltv_id.split('@')
+      const key = [channelId, feedId, channel.site, channel.site_id, channel.lang].join('_')
+      acc[key] = worker.getGuideSources()
+    })
+    return acc
+  }, {})
+
+  logger.info('preparing output...')
+  const output = channels.map((channel: Channel) => {
+    const [channelId, feedId] = channel.xmltv_id.split('@')
+    const key = [channelId, feedId, channel.site, channel.site_id, channel.lang].join('_')
+
+    return new ApiGuide({
+      channel: channelId,
+      feed: feedId,
+      site: channel.site,
+      site_id: channel.site_id,
+      site_name: channel.name,
+      lang: channel.lang,
+      sources: guidesMap[key] || []
+    })
+  })
+
+  logger.info('saving output...')
+  const outputFilename = 'guides.json'
+  await saveOutput(outputFilename, output.toJSON())
+
+  logger.info(`saved to "${path.join(API_DIR, outputFilename)}"`)
+}
+
+main()
+
+async function saveOutput(filename: string, json: string) {
+  const apiStorage = new Storage(API_DIR)
+  await apiStorage.save(filename, json)
+}
+
+async function loadWorkers() {
+  const dataStorage = new Storage(DATA_DIR)
+
+  const workers = await dataStorage.json('workers.json')
+
+  return Array.isArray(workers) ? workers : []
+}
+
+async function loadChannels() {
+  const sitesStorage = new Storage(SITES_DIR)
+
+  const files: string[] = await sitesStorage.list('**/*.channels.xml')
+
+  const channels = new Collection<Channel>()
+  for (const filepath of files) {
+    const xml = await sitesStorage.load(filepath)
+    const parsedChannels = EPGGrabber.parseChannelsXML(xml)
+    const channelsFromXML = new Collection(parsedChannels).map(
+      (channel: epgGrabber.Channel) => new Channel(channel.toObject())
+    )
+
+    channelsFromXML.forEach((channel: Channel) => {
+      channels.add(channel)
+    })
+  }
+
+  return channels
+}

scripts/commands/guides/update.ts

@@ -1,114 +1,26 @@
 import { HTMLTableRow, HTMLTableDataItem, HTMLTableColumn } from '../../types/htmlTable'
-import epgGrabber, { EPGGrabber } from 'epg-grabber'
+import { Worker, WorkerData, WorkerGuideSource } from '../../models'
-import AxiosMockAdapter from 'axios-mock-adapter'
+import { DATA_DIR, ROOT_DIR } from '../../constants'
 import { Storage } from '@freearhey/storage-js'
-import { Channel, Worker } from '../../models'
 import { Collection } from '@freearhey/core'
-import { ROOT_DIR } from '../../constants'
 import { Logger } from '@freearhey/core'
 import { HTMLTable } from '../../core'
-import epgParser from 'epg-parser'
-import axios from 'axios'
 
 async function main() {
   const logger = new Logger({ level: process.env.NODE_ENV === 'test' ? -999 : 3 })
-  const rootStorage = new Storage(ROOT_DIR)
+  const dataStorage = new Storage(DATA_DIR)
-  const workers = new Map<string, Worker>()
 
-  logger.info('loading workers.txt...')
+  logger.info('loading workers.json...')
-  const workersTxt = await rootStorage.load('workers.txt')
+  const workers = await dataStorage.json('workers.json')
 
-  workersTxt.split('\r\n').forEach((host: string) => {
+  if (!Array.isArray(workers)) return
-    if (!host) return
-
-    const worker = new Worker({ host })
-
-    workers.set(host, worker)
-  })
-
-  for (const worker of workers.values()) {
-    logger.info(`processing "${worker.host}"...`)
-
-    const client = axios.create({
-      baseURL: worker.getBaseUrl(),
-      timeout: 60000
-    })
-
-    if (process.env.NODE_ENV === 'test') {
-      const mock = new AxiosMockAdapter(client)
-      if (worker.host === 'example.com') {
-        mock.onGet('worker.json').reply(404)
-      } else {
-        const testStorage = new Storage('tests/__data__/input/guides_update')
-        mock.onGet('worker.json').reply(200, await testStorage.load('worker.json'))
-        mock.onGet('channels.xml').reply(200, await testStorage.load('channels.xml'))
-        mock.onGet('guide.xml').reply(200, await testStorage.load('guide.xml'))
-      }
-    }
-
-    const workerJson = await client
-      .get('worker.json')
-      .then(res => res.data)
-      .catch(err => {
-        worker.status = err.status
-        logger.error(err.message)
-      })
-
-    if (!workerJson) {
-      worker.status = 'MISSING_WORKER_CONFIG'
-      logger.error('Unable to load "workers.json"')
-      continue
-    }
-
-    worker.channelsPath = workerJson.channels
-    worker.guidePath = workerJson.guide
-
-    if (!worker.channelsPath) {
-      worker.status = 'MISSING_CHANNELS_PATH'
-      logger.error('The "channels" property is missing from the workers config')
-      continue
-    }
-
-    if (!worker.guidePath) {
-      worker.status = 'MISSING_GUIDE_PATH'
-      logger.error('The "guide" property is missing from the workers config')
-      continue
-    }
-
-    const channelsXml = await client
-      .get(worker.channelsPath)
-      .then(res => res.data)
-      .catch(err => {
-        worker.status = err.status
-        logger.error(err.message)
-      })
-
-    if (!channelsXml) continue
-
-    const parsedChannels = EPGGrabber.parseChannelsXML(channelsXml)
-    worker.channels = new Collection(parsedChannels).map(
-      (channel: epgGrabber.Channel) => new Channel(channel.toObject())
-    )
-
-    const guideXml = await client
-      .get(worker.guidePath)
-      .then(res => res.data)
-      .catch(err => {
-        worker.status = err.status
-        logger.error(err.message)
-      })
-
-    if (!guideXml) continue
-
-    const parsedGuide = epgParser.parse(guideXml)
-    worker.lastUpdated = parsedGuide.date
-
-    worker.status = 'OK'
-  }
 
   logger.info('creating guides table...')
   const rows = new Collection<HTMLTableRow>()
-  workers.forEach((worker: Worker) => {
+  workers.forEach((data: WorkerData) => {
+    const worker = new Worker(data)
+
+    const sources = worker.getGuideSources()
     rows.add(
       new Collection<HTMLTableDataItem>([
         { value: worker.host },
@@ -116,16 +28,18 @@ async function main() {
         { value: worker.getChannelsCount().toString(), align: 'right' },
         { value: worker.getLastUpdated(), align: 'left' },
         {
-          value:
+          value: sources.length
-            worker.status === 'OK'
+            ? sources
-              ? `<a href="${worker.getChannelsUrl()}">${worker.channelsPath}</a><br><a href="${worker.getGuideUrl()}">${worker.guidePath}</a>`
+                .map((source: WorkerGuideSource) => `<a href="${source.url}">${source.format}</a>`)
-              : ''
+                .join(' | ')
+            : '-'
         }
       ])
     )
   })
 
   logger.info('updating guides.md...')
+  const rootStorage = new Storage(ROOT_DIR)
   const table = new HTMLTable(
     rows,
     new Collection<HTMLTableColumn>([

scripts/commands/workers/load.ts

@@ -0,0 +1,122 @@
+import epgGrabber, { EPGGrabber } from 'epg-grabber'
+import { DATA_DIR, ROOT_DIR } from '../../constants'
+import AxiosMockAdapter from 'axios-mock-adapter'
+import { Storage } from '@freearhey/storage-js'
+import { Channel, Worker } from '../../models'
+import { Collection } from '@freearhey/core'
+import { Logger } from '@freearhey/core'
+import epgParser from 'epg-parser'
+import axios from 'axios'
+import path from 'path'
+
+async function main() {
+  const logger = new Logger({ level: process.env.NODE_ENV === 'test' ? -999 : 3 })
+  const rootStorage = new Storage(ROOT_DIR)
+  const workers = new Map<string, Worker>()
+
+  logger.info('loading workers.txt...')
+  const workersTxt = await rootStorage.load('workers.txt')
+  const hosts = workersTxt.split('\r\n')
+
+  hosts.forEach((host: string) => {
+    if (!host) return
+
+    const worker = new Worker({ host })
+
+    workers.set(host, worker)
+  })
+
+  for (const worker of workers.values()) {
+    logger.info(`processing "${worker.host}"...`)
+
+    const client = axios.create({
+      baseURL: worker.getBaseUrl(),
+      timeout: 60000
+    })
+
+    if (process.env.NODE_ENV === 'test') {
+      const mock = new AxiosMockAdapter(client)
+      if (worker.host === 'example.com') {
+        mock.onGet('worker.json').reply(404)
+      } else {
+        const testStorage = new Storage('tests/__data__/input/workers_load')
+        mock.onGet('worker.json').reply(200, await testStorage.load('worker.json'))
+        mock.onGet('channels.xml').reply(200, await testStorage.load('channels.xml'))
+        mock.onGet('guide.xml').reply(200, await testStorage.load('guide.xml'))
+      }
+    }
+
+    const workerConfig = await client
+      .get('worker.json')
+      .then(res => res.data)
+      .catch(err => {
+        worker.setStatus(err.status)
+        logger.error(err.message)
+      })
+
+    if (!workerConfig) {
+      worker.setStatus('MISSING_WORKER_CONFIG')
+      logger.error('Unable to load "workers.json"')
+      continue
+    }
+
+    worker
+      .setChannelsPath(workerConfig.channels)
+      .setGuideXmlPath(
+        typeof workerConfig.guide === 'string' ? workerConfig.guide : workerConfig?.guide?.xml
+      )
+      .setGuideGzipPath(workerConfig?.guide?.gzip)
+      .setGuideJsonPath(workerConfig?.guide?.json)
+
+    if (!worker.channelsPath) {
+      worker.setStatus('MISSING_CHANNELS_PATH')
+      logger.error('The "channels" property is missing from the workers config')
+      continue
+    }
+
+    if (!worker.guideXmlPath) {
+      worker.setStatus('MISSING_GUIDE_XML_PATH')
+      logger.error('The "guide" property is missing from the workers config')
+      continue
+    }
+
+    const channelsXml = await client
+      .get(worker.channelsPath)
+      .then(res => res.data)
+      .catch(err => {
+        worker.status = err.status
+        logger.error(err.message)
+      })
+
+    if (!channelsXml) continue
+
+    const parsedChannels = EPGGrabber.parseChannelsXML(channelsXml)
+    worker.channels = new Collection(parsedChannels).map(
+      (channel: epgGrabber.Channel) => new Channel(channel.toObject())
+    )
+
+    const guideXml = await client
+      .get(worker.guideXmlPath)
+      .then(res => res.data)
+      .catch(err => {
+        worker.setStatus(err.status)
+        logger.error(err.message)
+      })
+
+    if (!guideXml) continue
+
+    const parsedGuide = epgParser.parse(guideXml)
+    worker.lastUpdated = parsedGuide.date
+
+    worker.setStatus('OK')
+  }
+
+  const output = [...workers.values()]
+  const dataStorage = new Storage(DATA_DIR)
+  const outputFilename = 'workers.json'
+  await dataStorage.save(outputFilename, JSON.stringify(output))
+
+  logger.info(`saved to "${path.join(DATA_DIR, outputFilename)}"`)
+}
+
+main()

scripts/default.config.js

@@ -5,6 +5,7 @@ export default {
   maxConnections: 1,
   curl: false,
   gzip: false,
+  json: false,
   debug: false,
   request: {
     maxContentLength: 5242880,

scripts/models/apiGuide.ts

@@ -0,0 +1,49 @@
+import { WorkerGuideSource } from './worker'
+
+export interface ApiGuideData {
+  channel: string | null
+  feed: string | null
+  site: string
+  site_id: string
+  site_name: string
+  lang: string | null
+  sources?: WorkerGuideSource[]
+}
+
+export class ApiGuide {
+  channel: string | null
+  feed: string | null
+  site: string
+  site_id: string
+  site_name: string
+  lang: string
+  sources: WorkerGuideSource[] = []
+
+  constructor(data: ApiGuideData) {
+    this.channel = data.channel || null
+    this.feed = data.feed || null
+    this.site = data.site
+    this.site_id = data.site_id
+    this.site_name = data.site_name
+    this.lang = data.lang || 'en'
+    this.sources = data.sources || []
+  }
+
+  addSource(source: ApiGuideSource): this {
+    this.sources.push(source)
+
+    return this
+  }
+
+  toJSON(): ApiGuideData {
+    return {
+      channel: this.channel,
+      feed: this.feed,
+      site: this.site,
+      site_id: this.site_id,
+      site_name: this.site_name,
+      lang: this.lang,
+      sources: this.sources
+    }
+  }
+}

scripts/models/channel.ts

@@ -1,22 +1,8 @@
-import { ChannelGuideObject } from '../types/channel'
 import * as epgGrabber from 'epg-grabber'
 import { SITES_DIR } from '../constants'
 import path from 'node:path'
 
 export class Channel extends epgGrabber.Channel {
-  getGuideObject(): ChannelGuideObject {
-    const [channelId, feedId] = this.xmltv_id.split('@')
-
-    return {
-      channel: channelId || null,
-      feed: feedId || null,
-      site: this.site,
-      site_id: this.site_id,
-      site_name: this.name,
-      lang: this.lang || 'en'
-    }
-  }
-
   getConfigPath(): string {
     return path.resolve(SITES_DIR, `${this.site}/${this.site}.config.js`)
   }

scripts/models/guide.ts

@@ -13,20 +13,23 @@ interface GuideData {
   channels: Collection<Channel>
   programs: Collection<Program>
   filepath: string
-  gzip: boolean
+  gzip: boolean | string
+  json: boolean | string
 }
 
 export class Guide {
   channels: Collection<Channel>
   programs: Collection<Program>
   filepath: string
-  gzip: boolean
+  gzip: boolean | string
+  json: boolean | string
 
   constructor(data: GuideData) {
     this.channels = data.channels
     this.programs = data.programs
     this.filepath = data.filepath
     this.gzip = data.gzip || false
+    this.json = data.json || false
   }
 
   addChannel(channel: Channel) {
@@ -35,8 +38,9 @@ export class Guide {
 
   toString() {
     const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString())
+    const headers = { date: currDate.format('YYYYMMDD') }
 
-    return EPGGrabber.generateXMLTV(this.channels.all(), this.programs.all(), currDate)
+    return EPGGrabber.generateXMLTV(this.channels.all(), this.programs.all(), headers)
   }
 
   async save({ logger }: { logger: Logger }) {
@@ -50,10 +54,23 @@ export class Guide {
 
     if (this.gzip) {
       const compressed = pako.gzip(xmltv)
-      const gzFilepath = `${this.filepath}.gz`
+      const gzFilepath = typeof this.gzip === 'string' ? this.gzip : `${this.filepath}.gz`
       const gzFilename = path.basename(gzFilepath)
       logger.info(`  saving to "${gzFilepath}"...`)
       await storage.save(gzFilename, compressed)
     }
+
+    if (this.json) {
+      const filename = path.basename(this.filepath).split('.')[0]
+      const jsonFilepath =
+        typeof this.json === 'string' ? this.json : path.join(dir, `${filename}.json`)
+      const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString())
+      const headers = { date: currDate.format('YYYYMMDD') }
+      const channels = this.channels.map((channel: Channel) => channel.toObject()).all()
+      const programs = this.programs.map((program: Program) => program.toObject()).all()
+      const json = JSON.stringify({ ...headers, channels, programs })
+      logger.info(`  saving to "${jsonFilepath}"...`)
+      await storage.save(jsonFilepath, json)
+    }
   }
 }

scripts/models/worker.ts

@@ -1,5 +1,6 @@
 import relativeTime from 'dayjs/plugin/relativeTime'
 import { Collection } from '@freearhey/core'
+import * as epgGrabber from 'epg-grabber'
 import { Channel } from './channel'
 import utc from 'dayjs/plugin/utc'
 import dayjs from 'dayjs'
@@ -7,20 +8,79 @@ import dayjs from 'dayjs'
 dayjs.extend(relativeTime)
 dayjs.extend(utc)
 
+export interface WorkerGuideSource {
+  host: string
+  format: string
+  url: string
+}
+
 export interface WorkerData {
   host: string
+  channelsPath?: string
+  channels?: epgGrabber.Types.ChannelData[]
+  guideXmlPath?: string
+  guideGzipPath?: string
+  guideJsonPath?: string
+  status?: string
+  lastUpdated?: string
 }
 
 export class Worker {
   host: string
   channelsPath?: string
-  guidePath?: string
   channels?: Collection<Channel>
+  guideXmlPath?: string
+  guideGzipPath?: string
+  guideJsonPath?: string
   status?: string
   lastUpdated?: string
 
   constructor(data: WorkerData) {
     this.host = data.host
+
+    this.update(data)
+  }
+
+  update(data: Partial<WorkerData>): this {
+    if (data.host) this.host = data.host
+    if (data.channelsPath) this.channelsPath = data.channelsPath
+    if (data.guideXmlPath) this.guideXmlPath = data.guideXmlPath
+    if (data.guideGzipPath) this.guideGzipPath = data.guideGzipPath
+    if (data.guideJsonPath) this.guideJsonPath = data.guideJsonPath
+    if (data.status) this.status = data.status
+    if (data.lastUpdated) this.lastUpdated = data.lastUpdated
+
+    if (data.channels) {
+      const channelInstances = data.channels.map(c => new Channel(c))
+      this.channels = new Collection(channelInstances)
+    }
+
+    return this
+  }
+
+  setChannelsPath(path: string): this {
+    this.channelsPath = path
+    return this
+  }
+
+  setGuideXmlPath(path: string): this {
+    this.guideXmlPath = path
+    return this
+  }
+
+  setGuideGzipPath(path: string): this {
+    this.guideGzipPath = path
+    return this
+  }
+
+  setGuideJsonPath(path: string): this {
+    this.guideJsonPath = path
+    return this
+  }
+
+  setStatus(status: string): this {
+    this.status = status
+    return this
   }
 
   getBaseUrl(): string {
@@ -41,10 +101,26 @@ export class Worker {
     return url.href
   }
 
-  getGuideUrl(): string {
+  getGuideXmlUrl(): string {
-    if (!this.guidePath) return ''
+    if (!this.guideXmlPath) return ''
 
-    const url = new URL(this.guidePath, this.getBaseUrl())
+    const url = new URL(this.guideXmlPath, this.getBaseUrl())
+
+    return url.href
+  }
+
+  getGuideGzipUrl(): string {
+    if (!this.guideGzipPath) return ''
+
+    const url = new URL(this.guideGzipPath, this.getBaseUrl())
+
+    return url.href
+  }
+
+  getGuideJsonUrl(): string {
+    if (!this.guideJsonPath) return ''
+
+    const url = new URL(this.guideJsonPath, this.getBaseUrl())
 
     return url.href
   }
@@ -65,9 +141,42 @@ export class Worker {
   getLastUpdated(): string {
     if (!this.lastUpdated) return '-'
 
-    let now = dayjs()
+    const now = dayjs.utc(process.env.CURR_DATE || new Date().toISOString())
-    if (process.env.NODE_ENV === 'test') now = dayjs.utc('2026-02-13')
 
     return dayjs.utc(this.lastUpdated).from(now)
   }
+
+  getGuideSources(): WorkerGuideSource[] {
+    const sources = []
+
+    if (this.guideXmlPath) {
+      const url = new URL(this.guideXmlPath, this.getBaseUrl())
+      sources.push({ host: this.host, url: url.href, format: 'XML' })
+    }
+
+    if (this.guideGzipPath) {
+      const url = new URL(this.guideGzipPath, this.getBaseUrl())
+      sources.push({ host: this.host, url: url.href, format: 'GZIP' })
+    }
+
+    if (this.guideJsonPath) {
+      const url = new URL(this.guideJsonPath, this.getBaseUrl())
+      sources.push({ host: this.host, url: url.href, format: 'JSON' })
+    }
+
+    return sources
+  }
+
+  toJSON(): WorkerData {
+    return {
+      host: this.host,
+      channelsPath: this.channelsPath,
+      channels: this.channels ? this.channels.map(c => c.toObject()).all() : [],
+      guideXmlPath: this.guideXmlPath,
+      guideGzipPath: this.guideGzipPath,
+      guideJsonPath: this.guideJsonPath,
+      status: this.status,
+      lastUpdated: this.lastUpdated
+    }
+  }
 }

scripts/types/channel.d.ts

@@ -1,8 +0,0 @@
-export interface ChannelGuideObject {
-  channel: string | null
-  feed: string | null
-  site: string
-  site_id: string
-  site_name: string
-  lang: string
-}

tests/__data__/expected/api_generate/guides.json

@@ -1 +0,0 @@
-[{"channel":"Channel1.us","feed":"SD","site":"example.com","site_id":"140","site_name":"Channel 1","lang":"en"},{"channel":"Channel2.us","feed":null,"site":"example.com","site_id":"142","site_name":"Channel 2","lang":"en"},{"channel":"Channel1.us","feed":"HD","site":"example.com","site_id":"140","site_name":"Channel 1","lang":"fr"},{"channel":"Channel3.us","feed":"HD","site":"example2.com","site_id":"150","site_name":"Channel 3","lang":"en"},{"channel":"Channel4.us","feed":null,"site":"example2.com","site_id":"152","site_name":"Channel 4","lang":"en"},{"channel":"Channel1.us","feed":null,"site":"example2.com","site_id":"140","site_name":"Channel 1","lang":"fr"}]

tests/__data__/expected/epg_grab/base.guide.xml

@@ -1,8 +1,8 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
-<channel id="Channel1.us@HD"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/><url>https://example.com</url></channel>
+<channel id="Channel1.us@HD"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/></channel>
-<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
+<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><lcn>36</lcn></channel>
-<channel id="140"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
+<channel id="140"><display-name>Channel 1</display-name></channel>
-<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example.com</url></channel>
+<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/></channel>
 <programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="140"><title lang="fr">Programme1 (example.com)</title></programme>
 <programme start="20221021043000 +0000" stop="20221021071000 +0000" channel="140"><title lang="fr">Programme1 (example.com)</title></programme>
 <programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us@HD"><title lang="en">Program1 (example.com)</title></programme>

tests/__data__/expected/epg_grab/custom_channels.guide.xml

@@ -1,10 +1,10 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
-<channel id="Channel1.us"><display-name>Custom Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/><url>https://example.com</url></channel>
+<channel id="Channel1.us"><display-name>Custom Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/></channel>
-<channel id="Channel1.us"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/><url>https://example2.com</url></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/></channel>
-<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/><url>https://example2.com</url></channel>
+<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/></channel>
-<channel id="Channel3.us@Wrong"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example2.com</url></channel>
+<channel id="Channel3.us@Wrong"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/></channel>
-<channel id="140"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
+<channel id="140"><display-name>Channel 1</display-name></channel>
-<channel id="Channel2.us"><display-name>Custom Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url></channel>
+<channel id="Channel2.us"><display-name>Custom Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/></channel>
 <programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221021043100 +0000" stop="20221021071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221021044000 +0000" stop="20221021071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example2.com)</title></programme>

tests/__data__/expected/epg_grab/gzip/guide.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
-<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example2.com</url></channel>
+<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/></channel>
-<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/><url>https://example2.com</url></channel>
+<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/></channel>
-<channel id="Channel1.us"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/><url>https://example2.com</url></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/></channel>
 <programme start="20221020044000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example2.com)</title></programme>
 <programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel3.us"><title lang="en">Program1 (example2.com)</title></programme>
 <programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel4.us@HD"><title lang="en">Program1 (example2.com)</title></programme>

tests/__data__/expected/epg_grab/json/guide.json

@@ -0,0 +1 @@
+{"date":"20221020","channels":[{"xmltv_id":"Channel3.us","name":"Channel 3","site":"example2.com","site_id":"150","lang":"en","logo":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png","url":null,"lcn":null,"index":0},{"xmltv_id":"Channel4.us@HD","name":"Channel 4","site":"example2.com","site_id":"152","lang":"en","logo":"https://i.imgur.com/BPzH88J.png","url":null,"lcn":null,"index":1},{"xmltv_id":"Channel1.us","name":"Channel 1","site":"example2.com","site_id":"140","lang":"fr","logo":"https://i.imgur.com/GPzH88J.png","url":null,"lcn":null,"index":2}],"programs":[{"site":"example2.com","start":1666240800000,"stop":1666249800000,"channel":"Channel1.us","titles":[{"value":"Programme1 (example2.com)","lang":"fr"}],"subTitles":[],"descriptions":[],"date":0,"categories":[],"keywords":[],"languages":[],"origLanguages":[],"length":[],"urls":[],"countries":[],"episodeNumbers":[],"video":{"present":"","colour":"","aspect":"","quality":""},"audio":{"present":"","stereo":""},"previouslyShown":[],"premiere":[],"lastChance":[],"new":false,"subtitles":[],"ratings":[],"starRatings":[],"reviews":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"images":[],"icons":[]},{"site":"example2.com","start":1666240260000,"stop":1666249800000,"channel":"Channel3.us","titles":[{"value":"Program1 (example2.com)","lang":"en"}],"subTitles":[],"descriptions":[],"date":0,"categories":[],"keywords":[],"languages":[],"origLanguages":[],"length":[],"urls":[],"countries":[],"episodeNumbers":[],"video":{"present":"","colour":"","aspect":"","quality":""},"audio":{"present":"","stereo":""},"previouslyShown":[],"premiere":[],"lastChance":[],"new":false,"subtitles":[],"ratings":[],"starRatings":[],"reviews":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"images":[],"icons":[]},{"site":"example2.com","start":1666240260000,"stop":1666249800000,"channel":"Channel4.us@HD","titles":[{"value":"Program1 (example2.com)","lang":"en"}],"subTitles":[],"descriptions":[],"date":0,"categories":[],"keywords":[],"languages":[],"origLanguages":[],"length":[],"urls":[],"countries":[],"episodeNumbers":[],"video":{"present":"","colour":"","aspect":"","quality":""},"audio":{"present":"","stereo":""},"previouslyShown":[],"premiere":[],"lastChance":[],"new":false,"subtitles":[],"ratings":[],"starRatings":[],"reviews":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"images":[],"icons":[]}]}

tests/__data__/expected/epg_grab/json/guide.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
+<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/></channel>
+<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/></channel>
+<programme start="20221020044000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example2.com)</title></programme>
+<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel3.us"><title lang="en">Program1 (example2.com)</title></programme>
+<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel4.us@HD"><title lang="en">Program1 (example2.com)</title></programme>
+</tv>

tests/__data__/expected/epg_grab/lang/fr/guide.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
-<channel id="140"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
+<channel id="140"><display-name>Channel 1</display-name></channel>
 <programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="140"><title lang="fr">Programme1 (example.com)</title></programme>
 <programme start="20221021043000 +0000" stop="20221021071000 +0000" channel="140"><title lang="fr">Programme1 (example.com)</title></programme>
 </tv>

tests/__data__/expected/epg_grab/multilang.guide.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
-<channel id="140"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
+<channel id="140"><display-name>Channel 1</display-name></channel>
-<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example.com</url></channel>
+<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/></channel>
 <programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="140"><title lang="fr">Programme1 (example.com)</title></programme>
 <programme start="20221021043000 +0000" stop="20221021071000 +0000" channel="140"><title lang="fr">Programme1 (example.com)</title></programme>
 <programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>

tests/__data__/expected/epg_grab/multiple_channels.guide.xml

@@ -1,11 +1,11 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
-<channel id="Channel1.us@HD"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/><url>https://example.com</url></channel>
+<channel id="Channel1.us@HD"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/></channel>
-<channel id="Channel1.us"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/><url>https://example2.com</url></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/></channel>
-<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/><url>https://example2.com</url></channel>
+<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/></channel>
-<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example2.com</url></channel>
+<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/></channel>
-<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example.com</url></channel>
+<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/></channel>
-<channel id="140"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
+<channel id="140"><display-name>Channel 1</display-name></channel>
-<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
+<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><lcn>36</lcn></channel>
 <programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us@HD"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221021043100 +0000" stop="20221021071000 +0000" channel="Channel1.us@HD"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221021044000 +0000" stop="20221021071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example2.com)</title></programme>

tests/__data__/expected/epg_grab/wildcard/example.com/en/guide.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
-<channel id="Channel1.us@HD"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/><url>https://example.com</url></channel>
+<channel id="Channel1.us@HD"><display-name>Channel 1</display-name><icon src="https://i.imgur.com/GPzH88J.png"/></channel>
-<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
+<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><lcn>36</lcn></channel>
 <programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us@HD"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221021043100 +0000" stop="20221021071000 +0000" channel="Channel1.us@HD"><title lang="en">Program1 (example.com)</title></programme>
 </tv>

tests/__data__/expected/epg_grab/wildcard/example.com/fr/guide.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
-<channel id="140"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
+<channel id="140"><display-name>Channel 1</display-name></channel>
 <programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="140"><title lang="fr">Programme1 (example.com)</title></programme>
 <programme start="20221021043000 +0000" stop="20221021071000 +0000" channel="140"><title lang="fr">Programme1 (example.com)</title></programme>
 </tv>

tests/__data__/expected/epg_grab/wildcard/example.com/it/guide.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
-<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example.com</url></channel>
+<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/></channel>
 <programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
 <programme start="20221021043100 +0000" stop="20221021071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
 </tv>

tests/__data__/expected/guides_export/guides.json

@@ -0,0 +1 @@
+[{"channel":"Channel1.us","feed":"SD","site":"example.com","site_id":"140","site_name":"Channel 1","lang":"en","sources":[]},{"channel":"Channel2.us","feed":null,"site":"example.com","site_id":"142","site_name":"Channel 2","lang":"en","sources":[]},{"channel":"Channel1.us","feed":"HD","site":"example.com","site_id":"140","site_name":"Channel 1","lang":"fr","sources":[]},{"channel":"Channel3.us","feed":"HD","site":"example2.com","site_id":"150","site_name":"Channel 3","lang":"en","sources":[]},{"channel":"Channel4.us","feed":null,"site":"example2.com","site_id":"152","site_name":"Channel 4","lang":"en","sources":[]},{"channel":"Channel1.us","feed":null,"site":"example2.com","site_id":"140","site_name":"Channel 1","lang":"fr","sources":[]}]

tests/__data__/expected/guides_update/GUIDES.md

@@ -5,8 +5,8 @@
     <tr><th align="left">Host</th><th align="left">Status</th><th align="left">Channels</th><th align="left">Last Updated</th><th align="left">Links</th></tr>
   </thead>
   <tbody>
-    <tr><td>example.com</td><td align="center">🔴</td><td align="right">0</td><td align="left">-</td><td></td></tr>
+    <tr><td>example.com</td><td align="center">🔴</td><td align="right">0</td><td align="left">-</td><td>-</td></tr>
-    <tr><td>worker-9dd4.onrender.com</td><td align="center">🟢</td><td align="right">1</td><td align="left">a day ago</td><td><a href="https://worker-9dd4.onrender.com/channels.xml">channels.xml</a><br><a href="https://worker-9dd4.onrender.com/guide.xml">guide.xml</a></td></tr>
+    <tr><td>worker-9dd4.onrender.com</td><td align="center">🟢</td><td align="right">1</td><td align="left">a day ago</td><td><a href="https://worker-9dd4.onrender.com/guide.xml">XML</a> | <a href="https://worker-9dd4.onrender.com/guide.xml.gz">GZIP</a> | <a href="https://worker-9dd4.onrender.com/guide.json">JSON</a></td></tr>
   </tbody>
 </table>
 

tests/__data__/expected/workers_load/workers.json

@@ -0,0 +1 @@
+[{"host":"example.com","channels":[],"status":"MISSING_WORKER_CONFIG"},{"host":"worker-9dd4.onrender.com","channelsPath":"channels.xml","channels":[{"xmltv_id":"ANT1Europe.gr@SD","name":"ANT1 Europe","site":"antennaeurope.gr","site_id":"#","lang":"el","logo":null,"url":null,"lcn":null,"index":0}],"guideXmlPath":"guide.xml","guideGzipPath":"guide.xml.gz","guideJsonPath":"guide.json","status":"OK","lastUpdated":"2022-10-03T00:00:00.000Z"}]

tests/__data__/input/guides_export/sites/example.com/example.com.channels.xml

@@ -1,6 +1,6 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
-<channels site="example.com">
+<channels site="example.com">
-  <channel lang="en" xmltv_id="Channel1.us@SD" site_id="140">Channel 1</channel>
+  <channel lang="en" xmltv_id="Channel1.us@SD" site_id="140">Channel 1</channel>
-  <channel lang="en" xmltv_id="Channel2.us" site_id="142" logo="https://i.imgur.com/qmRnD0M.png">Channel 2</channel>
+  <channel lang="en" xmltv_id="Channel2.us" site_id="142" logo="https://i.imgur.com/qmRnD0M.png">Channel 2</channel>
-  <channel lang="fr" xmltv_id="Channel1.us@HD" site_id="140">Channel 1</channel>
+  <channel lang="fr" xmltv_id="Channel1.us@HD" site_id="140">Channel 1</channel>
 </channels>

tests/__data__/input/guides_export/sites/example2.com/example2.com.channels.xml

@@ -1,6 +1,6 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
-<channels site="example2.com">
+<channels site="example2.com">
-  <channel lang="en" xmltv_id="Channel3.us@HD" site_id="150">Channel 3</channel>
+  <channel lang="en" xmltv_id="Channel3.us@HD" site_id="150">Channel 3</channel>
-  <channel lang="en" xmltv_id="Channel4.us" site_id="152">Channel 4</channel>
+  <channel lang="en" xmltv_id="Channel4.us" site_id="152">Channel 4</channel>
-  <channel lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
+  <channel lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
 </channels>

tests/__data__/input/guides_update/worker.json

@@ -1,4 +0,0 @@
-{
-  "channels": "channels.xml",
-  "guide": "guide.xml"
-}

tests/__data__/input/guides_update/workers.json

@@ -0,0 +1,29 @@
+[
+  {
+    "host": "example.com",
+    "channels": [],
+    "status": "MISSING_WORKER_CONFIG"
+  },
+  {
+    "host": "worker-9dd4.onrender.com",
+    "channelsPath": "channels.xml",
+    "channels": [
+      {
+        "xmltv_id": "ANT1Europe.gr@SD",
+        "name": "ANT1 Europe",
+        "site": "antennaeurope.gr",
+        "site_id": "#",
+        "lang": "el",
+        "logo": null,
+        "url": null,
+        "lcn": null,
+        "index": 0
+      }
+    ],
+    "guideXmlPath": "guide.xml",
+    "guideGzipPath": "guide.xml.gz",
+    "guideJsonPath": "guide.json",
+    "status": "OK",
+    "lastUpdated": "2022-10-03T00:00:00.000Z"
+  }
+]

tests/__data__/input/workers_load/guide.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?><tv date="20260212">
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20221003">
 <channel id="ANT1Europe.gr@SD"><display-name>ANT1 Europe</display-name><icon src="https://i.imgur.com/DTfwJGM.png"/><url>https://antennaeurope.gr</url></channel>
 <programme start="20260212040000 +0000" stop="20260212070000 +0000" channel="ANT1Europe.gr@SD"><title lang="el">ΚΑΛΗΜΕΡΑ ΕΛΛΑΔΑ</title></programme>
 <programme start="20260212070000 +0000" stop="20260212103000 +0000" channel="ANT1Europe.gr@SD"><title lang="el">ΤΟ ΠΡΩΙΝΟ</title></programme>

tests/__data__/input/workers_load/worker.json

@@ -0,0 +1,8 @@
+{
+  "channels": "channels.xml",
+  "guide": {
+    "xml": "guide.xml",
+    "gzip": "guide.xml.gz",
+    "json": "guide.json"
+  }
+}

tests/commands/epg/grab.test.ts

@@ -90,21 +90,20 @@ describe('epg:grab', () => {
 
   it('can grab epg with gzip option enabled', () => {
     const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example2.com/example2.com.channels.xml --output="${path.resolve(
-      'tests/__data__/output/guides/gzip.guide.xml'
+      'tests/__data__/output/guides/guide.xml'
-    )}" --gzip `
+    )}" --gzip`
     const stdout = execSync(cmd, { encoding: 'utf8' })
     if (process.env.DEBUG === 'true') console.log(cmd, stdout)
 
-    expect(content('tests/__data__/output/guides/gzip.guide.xml')).toEqual(
+    expect(content('tests/__data__/output/guides/guide.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/gzip.guide.xml')
+      content('tests/__data__/expected/epg_grab/gzip/guide.xml')
     )
 
-    const outputString = pako.ungzip(
+    const outputString = pako.ungzip(fs.readFileSync('tests/__data__/output/guides/guide.xml.gz'), {
-      fs.readFileSync('tests/__data__/output/guides/gzip.guide.xml.gz'),
+      to: 'string'
-      { to: 'string' }
+    })
-    )
     const expectedString = pako.ungzip(
-      fs.readFileSync('tests/__data__/expected/epg_grab/gzip.guide.xml.gz'),
+      fs.readFileSync('tests/__data__/expected/epg_grab/gzip/guide.xml.gz'),
       { to: 'string' }
     )
 
@@ -113,6 +112,66 @@ describe('epg:grab', () => {
 
     expect(output).toEqual(expected)
   })
+
+  it('can grab epg with gzip option enabled', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example2.com/example2.com.channels.xml --output="${path.resolve(
+      'tests/__data__/output/guides/guide.xml'
+    )}" --gzip="${path.resolve('tests/__data__/output/guides/custom.xml.gz')}"`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guides/guide.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/gzip/guide.xml')
+    )
+
+    const outputString = pako.ungzip(
+      fs.readFileSync('tests/__data__/output/guides/custom.xml.gz'),
+      {
+        to: 'string'
+      }
+    )
+    const expectedString = pako.ungzip(
+      fs.readFileSync('tests/__data__/expected/epg_grab/gzip/guide.xml.gz'),
+      { to: 'string' }
+    )
+
+    const output = new Set(outputString.split('\r\n'))
+    const expected = new Set(expectedString.split('\r\n'))
+
+    expect(output).toEqual(expected)
+  })
+
+  it('can grab epg with json option enabled', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example2.com/example2.com.channels.xml --output="${path.resolve(
+      'tests/__data__/output/guides/guide.xml'
+    )}" --json`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guides/guide.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/json/guide.xml')
+    )
+
+    expect(content('tests/__data__/output/guides/guide.json')).toEqual(
+      content('tests/__data__/expected/epg_grab/json/guide.json')
+    )
+  })
+
+  it('can grab epg with json option enabled', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example2.com/example2.com.channels.xml --output="${path.resolve(
+      'tests/__data__/output/guides/guide.xml'
+    )}" --json="${path.resolve('tests/__data__/output/guides/custom.json')}"`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guides/guide.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/json/guide.xml')
+    )
+
+    expect(content('tests/__data__/output/guides/custom.json')).toEqual(
+      content('tests/__data__/expected/epg_grab/json/guide.json')
+    )
+  })
 })
 
 function content(filepath: string) {

tests/commands/guides/export.test.ts

@@ -1,27 +1,28 @@
-import { execSync } from 'child_process'
+import { execSync } from 'child_process'
-import fs from 'fs-extra'
+import fs from 'fs-extra'
-import { pathToFileURL } from 'node:url'
+import { pathToFileURL } from 'node:url'
-
+
-const ENV_VAR = 'cross-env SITES_DIR=tests/__data__/input/api_generate/sites API_DIR=tests/__data__/output'
+const ENV_VAR =
-
+  'cross-env SITES_DIR=tests/__data__/input/guides_export/sites API_DIR=tests/__data__/output'
-beforeEach(() => {
+
-  fs.emptyDirSync('tests/__data__/output')
+beforeEach(() => {
-})
+  fs.emptyDirSync('tests/__data__/output')
-
+})
-describe('api:generate', () => {
+
-  it('can generate guides.json', () => {
+describe('guides:export', () => {
-    const cmd = `${ENV_VAR} npm run api:generate`
+  it('can generate guides.json', () => {
-    const stdout = execSync(cmd, { encoding: 'utf8' })
+    const cmd = `${ENV_VAR} npm run guides:export`
-    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+    const stdout = execSync(cmd, { encoding: 'utf8' })
-
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
-    expect(content('tests/__data__/output/guides.json')).toEqual(
+
-      content('tests/__data__/expected/api_generate/guides.json')
+    expect(content('tests/__data__/output/guides.json')).toEqual(
-    )
+      content('tests/__data__/expected/guides_export/guides.json')
-  })
+    )
-})
+  })
-
+})
-function content(filepath: string) {
+
-  return fs.readFileSync(pathToFileURL(filepath), {
+function content(filepath: string) {
-    encoding: 'utf8'
+  return fs.readFileSync(pathToFileURL(filepath), {
-  })
+    encoding: 'utf8'
-}
+  })
+}

tests/commands/guides/update.test.ts

@@ -2,22 +2,24 @@ import { execSync } from 'child_process'
 import { pathToFileURL } from 'node:url'
 import fs from 'fs-extra'
 
-const ENV_VAR = 'cross-env ROOT_DIR=tests/__data__/output'
+const ENV_VAR =
+  'cross-env CURR_DATE=2022-10-04 ROOT_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/guides_update'
 
 beforeEach(() => {
   fs.emptyDirSync('tests/__data__/output')
-  fs.copySync('tests/__data__/input/guides_update/workers.txt', 'tests/__data__/output/workers.txt')
 })
 
-it('can update GUIDES.md', () => {
+describe('guides:update', () => {
-  const cmd = `${ENV_VAR} npm run guides:update`
+  it('can update GUIDES.md', () => {
+    const cmd = `${ENV_VAR} npm run guides:update`
 
-  const stdout = execSync(cmd, { encoding: 'utf8' })
+    const stdout = execSync(cmd, { encoding: 'utf8' })
-  if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
 
-  expect(content('tests/__data__/output/GUIDES.md')).toEqual(
+    expect(content('tests/__data__/output/GUIDES.md')).toEqual(
-    content('tests/__data__/expected/guides_update/GUIDES.md')
+      content('tests/__data__/expected/guides_update/GUIDES.md')
-  )
+    )
+  })
 })
 
 function content(filepath: string) {

tests/commands/workers/load.test.ts

@@ -0,0 +1,32 @@
+import { execSync } from 'child_process'
+import { pathToFileURL } from 'node:url'
+import fs from 'fs-extra'
+
+const ENV_VAR =
+  'cross-env CURR_DATE=2022-10-04 ROOT_DIR=tests/__data__/output DATA_DIR=tests/__data__/output/'
+
+beforeEach(() => {
+  fs.emptyDirSync('tests/__data__/output')
+  fs.copySync('tests/__data__/input/workers_load/workers.txt', 'tests/__data__/output/workers.txt')
+})
+
+describe('workers:load', () => {
+  it('can create workers.json', () => {
+    const cmd = `${ENV_VAR} npm run workers:load`
+
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/workers.json')).toEqual(
+      content('tests/__data__/expected/workers_load/workers.json')
+    )
+  })
+})
+
+function content(filepath: string) {
+  const data = fs.readFileSync(pathToFileURL(filepath), {
+    encoding: 'utf8'
+  })
+
+  return JSON.stringify(data)
+}

tsconfig.json

@@ -13,6 +13,6 @@
     "allowJs": true,
     "outDir": "./dist"
   },
-  "include": ["scripts/**/*", "src/**/*"],
+  "include": ["scripts/**/*", "tests/**/*"],
   "exclude": ["node_modules", "dist"]
 }