Merge pull request #2814 from iptv-org/patch-2025.07.1

Patch 2025.07.1

.github/workflows/check.yml

@@ -18,22 +18,23 @@ jobs:
       - name: changed files
         id: files
         run: |
+          git fetch origin master:master
           JS_ANY_CHANGED=false
-          JS_ALL_CHANGED_FILES=$(git diff --name-only tests/**/*.js tests/**/*.ts scripts/**/*.js scripts/**/*.mts scripts/**/*.ts sites/**/*.js sites/**/*.ts | tr '\n' ' ')
+          JS_ALL_CHANGED_FILES=$(git diff --name-only master -- tests/**/*.js tests/**/*.ts scripts/**/*.js scripts/**/*.mts scripts/**/*.ts sites/**/*.js sites/**/*.ts | tr '\n' ' ')
           if [ -n "${JS_ALL_CHANGED_FILES}" ]; then
             JS_ANY_CHANGED=true
           fi
           echo "js_all_changed_files=$JS_ALL_CHANGED_FILES" >> "$GITHUB_OUTPUT"
           echo "js_any_changed=$JS_ANY_CHANGED" >> "$GITHUB_OUTPUT"
           CHANNELS_ANY_CHANGED=false
-          CHANNELS_ALL_CHANGED_FILES=$(git diff --name-only sites/**/*.channels.xml | tr '\n' ' ')
+          CHANNELS_ALL_CHANGED_FILES=$(git diff --name-only master --  sites/**/*.channels.xml | tr '\n' ' ')
           if [ -n "${CHANNELS_ALL_CHANGED_FILES}" ]; then
             CHANNELS_ANY_CHANGED=true
           fi
           echo "channels_all_changed_files=$CHANNELS_ALL_CHANGED_FILES" >> "$GITHUB_OUTPUT"
           echo "channels_any_changed=$CHANNELS_ANY_CHANGED" >> "$GITHUB_OUTPUT"
       - uses: actions/setup-node@v4
-        if: ${{ !env.ACT && (steps.files.outputs.js_any_changed == 'true' || steps.files.outputs.channels_any_changed == 'true') }}
+        if: ${{ steps.files.outputs.js_any_changed == 'true' || steps.files.outputs.channels_any_changed == 'true' }}
         with:
           node-version: 22
           cache: 'npm'

.github/workflows/update.yml

@@ -7,14 +7,14 @@ jobs:
   main:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: tibdex/github-app-token@v1.8.2
         if: ${{ !env.ACT }}
         id: create-app-token
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PRIVATE_KEY }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         if: ${{ !env.ACT }}
         with:
           token: ${{ steps.create-app-token.outputs.token }}
@@ -22,8 +22,7 @@ jobs:
         run: |
           git config user.name "iptv-bot[bot]"
           git config user.email "84861620+iptv-bot[bot]@users.noreply.github.com"
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
-        if: ${{ !env.ACT }}
         with:
           node-version: 22
           cache: 'npm'

README.md

@@ -65,6 +65,7 @@ Options:
   --days <days>                 Number of days for which the program will be loaded (defaults to the value from the site config)
   --maxConnections <number>     Number of concurrent requests (default: 1)
   --gzip                        Specifies whether or not to create a compressed version of the guide (default: false)
+  --curl                        Display each request as CURL (default: false)
 ```
 
 ### Parallel downloading
@@ -97,19 +98,15 @@ npm run grab --- --channels=path/to/custom.channels.xml
 
 ### Run on schedule
 
-To download the guide on a schedule, you can use the included process manager. Just run it with desire [cron expression](https://crontab.guru/) and the `grab` options:
+If you want to download guides on a schedule, you can use [cron](https://en.wikipedia.org/wiki/Cron) or any other task scheduler. Currently, we use a tool called `chronos` for this purpose.
+
+To start it, you only need to specify the necessary `grab` command and [cron expression](https://crontab.guru/):
 
 ```sh
-npx pm2 start npm --no-autorestart --cron-restart="0 0,12 * * *" -- run grab --- --site=example.com
+npx chronos --execute="npm run grab --- --site=example.com" --pattern="0 0,12 * * *" --log
 ```
 
-To track the process, you can use the command:
+For more info go to [chronos](https://github.com/freearhey/chronos) documentation.
-
-```sh
-npx pm2 logs
-```
-
-For more info go to [pm2](https://pm2.keymetrics.io/docs/usage/quick-start/) documentation.
 
 ### Access the guide by URL
 
@@ -186,6 +183,7 @@ docker run \
 -e CRON_SCHEDULE="0 0,12 * * *" \
 -e MAX_CONNECTIONS=10 \
 -e GZIP=true \
+-e CURL=true \
 -e PROXY="socks5://127.0.0.1:1234" \
 -e DAYS=14 \
 -e TIMEOUT=5 \
@@ -198,6 +196,7 @@ iptv-org/epg
 | CRON_SCHEDULE   | A [cron expression](https://crontab.guru/) describing the schedule of the guide loadings (default: "0 0 \* \* \*") |
 | MAX_CONNECTIONS | Limit on the number of concurrent requests (default: 1)                                                            |
 | GZIP            | Boolean value indicating whether to create a compressed version of the guide (default: false)                      |
+| CURL            | Display each request as CURL (default: false)                                                                      |
 | PROXY           | Use the specified proxy                                                                                            |
 | DAYS            | Number of days for which the guide will be loaded (defaults to the value from the site config)                     |
 | TIMEOUT         | Timeout for each request in milliseconds (default: 0)                                                              |

package.json

@@ -18,7 +18,6 @@
     "prepare": "husky"
   },
   "private": true,
-  "author": "Arhey",
   "license": "UNLICENSED",
   "jest": {
     "setupFiles": [
@@ -40,27 +39,28 @@
     "@alex_neo/jest-expect-message": "^1.0.5",
     "@eslint/eslintrc": "^3.3.1",
     "@eslint/js": "^9.30.0",
-    "@freearhey/core": "^0.8.2",
+    "@freearhey/chronos": "^0.0.1",
+    "@freearhey/core": "^0.10.2",
     "@freearhey/search-js": "^0.1.2",
     "@ntlab/sfetch": "^1.2.0",
-    "@octokit/core": "^7.0.2",
+    "@octokit/core": "^7.0.3",
     "@octokit/plugin-paginate-rest": "^13.1.1",
     "@octokit/plugin-rest-endpoint-methods": "^16.0.0",
-    "@swc/core": "^1.12.7",
+    "@swc/core": "^1.13.0",
-    "@swc/jest": "^0.2.38",
+    "@swc/jest": "^0.2.39",
     "@types/cli-progress": "^3.11.6",
     "@types/fs-extra": "^11.0.4",
     "@types/inquirer": "^9.0.8",
     "@types/jest": "^30.0.0",
     "@types/langs": "^2.0.5",
     "@types/lodash": "^4.17.19",
-    "@types/node": "^24.0.7",
+    "@types/node": "^24.0.14",
     "@types/node-cleanup": "^2.1.5",
     "@types/numeral": "^2.0.5",
     "@typescript-eslint/eslint-plugin": "^8.35.0",
     "@typescript-eslint/parser": "^8.35.0",
     "axios": "^1.10.0",
-    "axios-cookiejar-support": "^6.0.2",
+    "axios-cookiejar-support": "^6.0.4",
     "chalk": "^5.4.1",
     "cheerio": "^1.1.0",
     "cli-progress": "^3.12.0",
@@ -70,23 +70,23 @@
     "csv-parser": "^3.2.0",
     "cwait": "^1.1.2",
     "dayjs": "^1.11.13",
-    "epg-grabber": "^0.38.0",
+    "epg-grabber": "^0.41.0",
     "epg-parser": "^0.3.1",
     "eslint": "^9.30.0",
-    "eslint-config-prettier": "^10.1.5",
+    "eslint-config-prettier": "^10.1.8",
-    "form-data": "^4.0.3",
+    "form-data": "^4.0.4",
     "fs-extra": "^11.3.0",
     "glob": "^11.0.3",
-    "globals": "^16.2.0",
+    "globals": "^16.3.0",
     "husky": "^9.1.7",
     "iconv-lite": "^0.6.3",
-    "inquirer": "^12.6.3",
+    "inquirer": "^12.7.0",
     "jest": "^30.0.3",
     "jest-offline": "^1.0.1",
     "langs": "^2.0.0",
     "libxml2-wasm": "^0.5.0",
     "lodash": "^4.17.21",
-    "luxon": "^3.6.1",
+    "luxon": "^3.7.1",
     "mockdate": "^3.0.5",
     "nedb-promises": "^6.2.3",
     "node-cleanup": "^2.1.2",
@@ -111,6 +111,5 @@
     "typescript": "^5.8.3",
     "unzipit": "^1.4.3",
     "wildcard-match": "^5.1.4"
-  },
+  }
-  "packageManager": "yarn@4.9.2"
 }

pm2.config.js

@@ -1,3 +1,9 @@
+const grab = process.env.SITE
+  ? `npm run grab -- --site=${process.env.SITE} ${
+      process.env.CLANG ? `--lang=${process.env.CLANG}` : ''
+    } --output=public/guide.xml`
+  : 'npm run grab -- --channels=channels.xml --output=public/guide.xml'
+
 module.exports = {
   apps: [
     {
@@ -9,15 +15,10 @@ module.exports = {
     },
     {
       name: 'grab',
-      script: process.env.SITE
+      script: `npx chronos -e "${grab}" -p "${process.env.CRON_SCHEDULE}" -l`,
-        ? `npm run grab -- --site=${process.env.SITE} ${
-            process.env.CLANG ? `--lang=${process.env.CLANG}` : ''
-          } --output=public/guide.xml`
-        : 'npm run grab -- --channels=channels.xml --output=public/guide.xml',
-      cron_restart: process.env.CRON_SCHEDULE || null,
       instances: 1,
       watch: false,
-      autorestart: false
+      autorestart: true
     }
   ]
 }

scripts/commands/api/generate.ts

@@ -1,17 +1,9 @@
-import { Logger, Storage, Collection } from '@freearhey/core'
+import { Logger, Collection, Storage } from '@freearhey/core'
-import { ChannelsParser } from '../../core'
-import path from 'path'
 import { SITES_DIR, API_DIR } from '../../constants'
+import { GuideChannel } from '../../models'
+import { ChannelsParser } from '../../core'
 import epgGrabber from 'epg-grabber'
-
+import path from 'path'
-type OutputItem = {
-  channel: string | null
-  feed: string | null
-  site: string
-  site_id: string
-  site_name: string
-  lang: string
-}
 
 async function main() {
   const logger = new Logger()
@@ -20,32 +12,25 @@ async function main() {
 
   logger.info('loading channels...')
   const sitesStorage = new Storage(SITES_DIR)
-  const parser = new ChannelsParser({ storage: sitesStorage })
+  const parser = new ChannelsParser({
-
+    storage: sitesStorage
-  let files: string[] = []
-  files = await sitesStorage.list('**/*.channels.xml')
-
-  let parsedChannels = new Collection()
-  for (const filepath of files) {
-    parsedChannels = parsedChannels.concat(await parser.parse(filepath))
-  }
-
-  logger.info(`  found ${parsedChannels.count()} channel(s)`)
-
-  const output = parsedChannels.map((channel: epgGrabber.Channel): OutputItem => {
-    const xmltv_id = channel.xmltv_id || ''
-    const [channelId, feedId] = xmltv_id.split('@')
-
-    return {
-      channel: channelId || null,
-      feed: feedId || null,
-      site: channel.site || '',
-      site_id: channel.site_id || '',
-      site_name: channel.name,
-      lang: channel.lang || ''
-    }
   })
 
+  const files: string[] = await sitesStorage.list('**/*.channels.xml')
+
+  const channels = new Collection()
+  for (const filepath of files) {
+    const channelList = await parser.parse(filepath)
+
+    channelList.channels.forEach((data: epgGrabber.Channel) => {
+      channels.add(new GuideChannel(data))
+    })
+  }
+
+  logger.info(`found ${channels.count()} channel(s)`)
+
+  const output = channels.map((channel: GuideChannel) => channel.toJSON())
+
   const apiStorage = new Storage(API_DIR)
   const outputFilename = 'guides.json'
   await apiStorage.save('guides.json', output.toJSON())

scripts/commands/api/load.ts

@@ -17,7 +17,8 @@ async function main() {
     loader.download('feeds.json'),
     loader.download('timezones.json'),
     loader.download('guides.json'),
-    loader.download('streams.json')
+    loader.download('streams.json'),
+    loader.download('logos.json')
   ])
 }
 

scripts/commands/channels/edit.ts

@@ -1,17 +1,17 @@
 import { Storage, Collection, Logger, Dictionary } from '@freearhey/core'
+import type { DataProcessorData } from '../../types/dataProcessor'
+import type { DataLoaderData } from '../../types/dataLoader'
+import { ChannelSearchableData } from '../../types/channel'
+import { Channel, ChannelList, Feed } from '../../models'
+import { DataProcessor, DataLoader } from '../../core'
 import { select, input } from '@inquirer/prompts'
-import { ChannelsParser, XML } from '../../core'
+import { ChannelsParser } from '../../core'
-import { Channel, Feed } from '../../models'
 import { DATA_DIR } from '../../constants'
 import nodeCleanup from 'node-cleanup'
+import sjs from '@freearhey/search-js'
+import epgGrabber from 'epg-grabber'
 import { Command } from 'commander'
 import readline from 'readline'
-import sjs from '@freearhey/search-js'
-import { DataProcessor, DataLoader } from '../../core'
-import type { DataLoaderData } from '../../types/dataLoader'
-import type { DataProcessorData } from '../../types/dataProcessor'
-import epgGrabber from 'epg-grabber'
-import { ChannelSearchableData } from '../../types/channel'
 
 type ChoiceValue = { type: string; value?: Feed | Channel }
 type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean }
@@ -34,11 +34,11 @@ program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(proc
 const filepath = program.args[0]
 const logger = new Logger()
 const storage = new Storage()
-let parsedChannels = new Collection()
+let channelList = new ChannelList({ channels: [] })
 
 main(filepath)
 nodeCleanup(() => {
-  save(filepath)
+  save(filepath, channelList)
 })
 
 export default async function main(filepath: string) {
@@ -51,18 +51,18 @@ export default async function main(filepath: string) {
   const dataStorage = new Storage(DATA_DIR)
   const loader = new DataLoader({ storage: dataStorage })
   const data: DataLoaderData = await loader.load()
-  const { feedsGroupedByChannelId, channels, channelsKeyById }: DataProcessorData =
+  const { channels, channelsKeyById, feedsGroupedByChannelId }: DataProcessorData =
     processor.process(data)
 
   logger.info('loading channels...')
   const parser = new ChannelsParser({ storage })
-  parsedChannels = await parser.parse(filepath)
+  channelList = await parser.parse(filepath)
-  const parsedChannelsWithoutId = parsedChannels.filter(
+  const parsedChannelsWithoutId = channelList.channels.filter(
     (channel: epgGrabber.Channel) => !channel.xmltv_id
   )
 
   logger.info(
-    `found ${parsedChannels.count()} channels (including ${parsedChannelsWithoutId.count()} without ID)`
+    `found ${channelList.channels.count()} channels (including ${parsedChannelsWithoutId.count()} without ID)`
   )
 
   logger.info('creating search index...')
@@ -73,10 +73,10 @@ export default async function main(filepath: string) {
 
   logger.info('starting...\n')
 
-  for (const parsedChannel of parsedChannelsWithoutId.all()) {
+  for (const channel of parsedChannelsWithoutId.all()) {
     try {
-      parsedChannel.xmltv_id = await selectChannel(
+      channel.xmltv_id = await selectChannel(
-        parsedChannel,
+        channel,
         searchIndex,
         feedsGroupedByChannelId,
         channelsKeyById
@@ -124,8 +124,8 @@ async function selectChannel(
     case 'channel': {
       const selectedChannel = selected.value
       if (!selectedChannel) return ''
-      const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId)
+      const selectedFeedId = await selectFeed(selectedChannel.id || '', feedsGroupedByChannelId)
-      if (selectedFeedId === '-') return selectedChannel.id
+      if (selectedFeedId === '-') return selectedChannel.id || ''
       return [selectedChannel.id, selectedFeedId].join('@')
     }
   }
@@ -153,7 +153,7 @@ async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary
     case 'feed':
       const selectedFeed = selected.value
       if (!selectedFeed) return ''
-      return selectedFeed.id
+      return selectedFeed.id || ''
   }
 
   return ''
@@ -205,10 +205,9 @@ function getFeedChoises(feeds: Collection): Choice[] {
   return choises
 }
 
-function save(filepath: string) {
+function save(filepath: string, channelList: ChannelList) {
   if (!storage.existsSync(filepath)) return
-  const xml = new XML(parsedChannels)
+  storage.saveSync(filepath, channelList.toString())
-  storage.saveSync(filepath, xml.toString())
   logger.info(`\nFile '${filepath}' successfully saved`)
 }
 

scripts/commands/channels/lint.mts

@@ -19,6 +19,7 @@ const xsd = `<?xml version="1.0" encoding="UTF-8"?>
       <xs:attribute use="required" ref="site_id"/>
       <xs:attribute name="xmltv_id" use="required" type="xs:string"/>
       <xs:attribute name="logo" type="xs:string"/>
+      <xs:attribute name="lcn" type="xs:string"/>
     </xs:complexType>
   </xs:element>
   <xs:attribute name="site">

scripts/commands/channels/parse.ts

@@ -1,8 +1,9 @@
-import { Logger, File, Collection, Storage } from '@freearhey/core'
+import { Logger, File, Storage } from '@freearhey/core'
-import { ChannelsParser, XML } from '../../core'
+import { ChannelsParser } from '../../core'
-import { Channel } from 'epg-grabber'
+import { ChannelList } from '../../models'
-import { Command } from 'commander'
 import { pathToFileURL } from 'node:url'
+import epgGrabber from 'epg-grabber'
+import { Command } from 'commander'
 
 const program = new Command()
 program
@@ -21,17 +22,25 @@ type ParseOptions = {
 const options: ParseOptions = program.opts()
 
 async function main() {
+  function isPromise(promise: object[] | Promise<object[]>) {
+    return (
+      !!promise &&
+      typeof promise === 'object' &&
+      typeof (promise as Promise<object[]>).then === 'function'
+    )
+  }
+
   const storage = new Storage()
-  const parser = new ChannelsParser({ storage })
   const logger = new Logger()
+  const parser = new ChannelsParser({ storage })
   const file = new File(options.config)
   const dir = file.dirname()
   const config = (await import(pathToFileURL(options.config).toString())).default
   const outputFilepath = options.output || `${dir}/${config.site}.channels.xml`
 
-  let channels = new Collection()
+  let channelList = new ChannelList({ channels: [] })
   if (await storage.exists(outputFilepath)) {
-    channels = await parser.parse(outputFilepath)
+    channelList = await parser.parse(outputFilepath)
   }
 
   const args: {
@@ -49,45 +58,31 @@ async function main() {
   if (isPromise(parsedChannels)) {
     parsedChannels = await parsedChannels
   }
-  parsedChannels = parsedChannels.map((channel: Channel) => {
+  parsedChannels = parsedChannels.map((channel: epgGrabber.Channel) => {
     channel.site = config.site
 
     return channel
   })
 
-  let output = new Collection()
+  const newChannelList = new ChannelList({ channels: [] })
-  parsedChannels.forEach((channel: Channel) => {
+  parsedChannels.forEach((channel: epgGrabber.Channel) => {
-    const found: Channel | undefined = channels.first(
+    if (!channel.site_id) return
-      (_channel: Channel) => _channel.site_id == channel.site_id
+
-    )
+    const found: epgGrabber.Channel | undefined = channelList.get(channel.site_id)
 
     if (found) {
       channel.xmltv_id = found.xmltv_id
       channel.lang = found.lang
     }
 
-    output.add(channel)
+    newChannelList.add(channel)
   })
 
-  output = output.orderBy([
+  newChannelList.sort()
-    (channel: Channel) => channel.lang || '_',
-    (channel: Channel) => (channel.xmltv_id ? channel.xmltv_id.toLowerCase() : '0'),
-    (channel: Channel) => channel.site_id
-  ])
 
-  const xml = new XML(output)
+  await storage.save(outputFilepath, newChannelList.toString())
-
-  await storage.save(outputFilepath, xml.toString())
 
   logger.info(`File '${outputFilepath}' successfully saved`)
 }
 
 main()
-
-function isPromise(promise: object[] | Promise<object[]>) {
-  return (
-    !!promise &&
-    typeof promise === 'object' &&
-    typeof (promise as Promise<object[]>).then === 'function'
-  )
-}

scripts/commands/channels/validate.ts

@@ -1,16 +1,18 @@
-import { Storage, Collection, Dictionary, File } from '@freearhey/core'
+import { ChannelsParser, DataLoader, DataProcessor } from '../../core'
-import { ChannelsParser } from '../../core'
+import { DataProcessorData } from '../../types/dataProcessor'
-import { Channel } from '../../models'
+import { Storage, Dictionary, File } from '@freearhey/core'
+import { DataLoaderData } from '../../types/dataLoader'
+import { ChannelList } from '../../models'
+import { DATA_DIR } from '../../constants'
+import epgGrabber from 'epg-grabber'
 import { program } from 'commander'
 import chalk from 'chalk'
 import langs from 'langs'
-import { DATA_DIR } from '../../constants'
-import epgGrabber from 'epg-grabber'
 
 program.argument('[filepath]', 'Path to *.channels.xml files to validate').parse(process.argv)
 
 type ValidationError = {
-  type: 'duplicate' | 'wrong_xmltv_id' | 'wrong_lang'
+  type: 'duplicate' | 'wrong_channel_id' | 'wrong_feed_id' | 'wrong_lang'
   name: string
   lang?: string
   xmltv_id?: string
@@ -19,12 +21,14 @@ type ValidationError = {
 }
 
 async function main() {
-  const parser = new ChannelsParser({ storage: new Storage() })
+  const processor = new DataProcessor()
-
   const dataStorage = new Storage(DATA_DIR)
-  const channelsData = await dataStorage.json('channels.json')
+  const loader = new DataLoader({ storage: dataStorage })
-  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const data: DataLoaderData = await loader.load()
-  const channelsGroupedById = channels.groupBy((channel: Channel) => channel.id)
+  const { channelsKeyById, feedsKeyByStreamId }: DataProcessorData = processor.process(data)
+  const parser = new ChannelsParser({
+    storage: new Storage()
+  })
 
   let totalFiles = 0
   let totalErrors = 0
@@ -35,11 +39,11 @@ async function main() {
     const file = new File(filepath)
     if (file.extension() !== 'xml') continue
 
-    const parsedChannels = await parser.parse(filepath)
+    const channelList: ChannelList = await parser.parse(filepath)
 
     const bufferBySiteId = new Dictionary()
     const errors: ValidationError[] = []
-    parsedChannels.forEach((channel: epgGrabber.Channel) => {
+    channelList.channels.forEach((channel: epgGrabber.Channel) => {
       const bufferId: string = channel.site_id
       if (bufferBySiteId.missing(bufferId)) {
         bufferBySiteId.set(bufferId, true)
@@ -54,12 +58,21 @@ async function main() {
       }
 
       if (!channel.xmltv_id) return
-      const [channelId] = channel.xmltv_id.split('@')
+      const [channelId, feedId] = channel.xmltv_id.split('@')
-      const foundChannel = channelsGroupedById.get(channelId)
+
+      const foundChannel = channelsKeyById.get(channelId)
       if (!foundChannel) {
-        errors.push({ type: 'wrong_xmltv_id', ...channel })
+        errors.push({ type: 'wrong_channel_id', ...channel })
         totalErrors++
       }
+
+      if (feedId) {
+        const foundFeed = feedsKeyByStreamId.get(channel.xmltv_id)
+        if (!foundFeed) {
+          errors.push({ type: 'wrong_feed_id', ...channel })
+          totalErrors++
+        }
+      }
     })
 
     if (errors.length) {

scripts/commands/epg/grab.ts

@@ -1,9 +1,10 @@
 import { Logger, Timer, Storage, Collection } from '@freearhey/core'
-import { Option, program } from 'commander'
 import { QueueCreator, Job, ChannelsParser } from '../../core'
+import { Option, program } from 'commander'
+import { SITES_DIR } from '../../constants'
 import { Channel } from 'epg-grabber'
 import path from 'path'
-import { SITES_DIR } from '../../constants'
+import { ChannelList } from '../../models'
 
 program
   .addOption(new Option('-s, --site <name>', 'Name of the site to parse'))
@@ -44,6 +45,7 @@ program
       .default(false)
       .env('GZIP')
   )
+  .addOption(new Option('--curl', 'Display each request as CURL').default(false).env('CURL'))
   .parse()
 
 export type GrabOptions = {
@@ -51,6 +53,7 @@ export type GrabOptions = {
   channels?: string
   output: string
   gzip: boolean
+  curl: boolean
   maxConnections: number
   timeout?: string
   delay?: string
@@ -85,31 +88,35 @@ async function main() {
     files = await storage.list(options.channels)
   }
 
-  let parsedChannels = new Collection()
+  let channels = new Collection()
   for (const filepath of files) {
-    parsedChannels = parsedChannels.concat(await parser.parse(filepath))
+    const channelList: ChannelList = await parser.parse(filepath)
+
+    channels = channels.concat(channelList.channels)
   }
+
   if (options.lang) {
-    parsedChannels = parsedChannels.filter((channel: Channel) => {
+    channels = channels.filter((channel: Channel) => {
       if (!options.lang || !channel.lang) return true
 
       return options.lang.includes(channel.lang)
     })
   }
-  logger.info(`  found ${parsedChannels.count()} channel(s)`)
+
+  logger.info(`  found ${channels.count()} channel(s)`)
 
   logger.info('run:')
-  runJob({ logger, parsedChannels })
+  runJob({ logger, channels })
 }
 
 main()
 
-async function runJob({ logger, parsedChannels }: { logger: Logger; parsedChannels: Collection }) {
+async function runJob({ logger, channels }: { logger: Logger; channels: Collection }) {
   const timer = new Timer()
   timer.start()
 
   const queueCreator = new QueueCreator({
-    parsedChannels,
+    channels,
     logger,
     options
   })

scripts/commands/sites/update.ts

@@ -1,21 +1,25 @@
 import { IssueLoader, HTMLTable, ChannelsParser } from '../../core'
 import { Logger, Storage, Collection } from '@freearhey/core'
+import { ChannelList, Issue, Site } from '../../models'
 import { SITES_DIR, ROOT_DIR } from '../../constants'
-import { Issue, Site } from '../../models'
 import { Channel } from 'epg-grabber'
 
 async function main() {
-  const logger = new Logger({ disabled: true })
+  const logger = new Logger({ level: -999 })
-  const loader = new IssueLoader()
+  const issueLoader = new IssueLoader()
   const sitesStorage = new Storage(SITES_DIR)
-  const channelsParser = new ChannelsParser({ storage: sitesStorage })
   const sites = new Collection()
 
+  logger.info('loading channels...')
+  const channelsParser = new ChannelsParser({
+    storage: sitesStorage
+  })
+
   logger.info('loading list of sites')
   const folders = await sitesStorage.list('*/')
 
   logger.info('loading issues...')
-  const issues = await loader.load()
+  const issues = await issueLoader.load()
 
   logger.info('putting the data together...')
   const brokenGuideReports = issues.filter(issue =>
@@ -33,19 +37,21 @@ async function main() {
 
     const files = await sitesStorage.list(`${domain}/*.channels.xml`)
     for (const filepath of files) {
-      const channels = await channelsParser.parse(filepath)
+      const channelList: ChannelList = await channelsParser.parse(filepath)
 
-      site.totalChannels += channels.count()
+      site.totalChannels += channelList.channels.count()
-      site.markedChannels += channels.filter((channel: Channel) => channel.xmltv_id).count()
+      site.markedChannels += channelList.channels
+        .filter((channel: Channel) => channel.xmltv_id)
+        .count()
     }
 
     sites.add(site)
   }
 
   logger.info('creating sites table...')
-  const data = new Collection()
+  const tableData = new Collection()
   sites.forEach((site: Site) => {
-    data.add([
+    tableData.add([
       { value: `<a href="sites/${site.domain}">${site.domain}</a>` },
       { value: site.totalChannels, align: 'right' },
       { value: site.markedChannels, align: 'right' },
@@ -55,7 +61,7 @@ async function main() {
   })
 
   logger.info('updating sites.md...')
-  const table = new HTMLTable(data.all(), [
+  const table = new HTMLTable(tableData.all(), [
     { name: 'Site', align: 'left' },
     { name: 'Channels<br>(total / with xmltv-id)', colspan: 2, align: 'left' },
     { name: 'Status', align: 'left' },

scripts/core/channelsParser.ts

@@ -1,5 +1,6 @@
 import { parseChannels } from 'epg-grabber'
-import { Storage, Collection } from '@freearhey/core'
+import { Storage } from '@freearhey/core'
+import { ChannelList } from '../models'
 
 type ChannelsParserProps = {
   storage: Storage
@@ -12,13 +13,10 @@ export class ChannelsParser {
     this.storage = storage
   }
 
-  async parse(filepath: string) {
+  async parse(filepath: string): Promise<ChannelList> {
-    let parsedChannels = new Collection()
-
     const content = await this.storage.load(filepath)
-    const channels = parseChannels(content)
+    const parsed = parseChannels(content)
-    parsedChannels = parsedChannels.concat(new Collection(channels))
 
-    return parsedChannels
+    return new ChannelList({ channels: parsed })
   }
 }

scripts/core/dataLoader.ts

@@ -49,7 +49,8 @@ export class DataLoader {
       feeds,
       timezones,
       guides,
-      streams
+      streams,
+      logos
     ] = await Promise.all([
       this.storage.json('countries.json'),
       this.storage.json('regions.json'),
@@ -61,7 +62,8 @@ export class DataLoader {
       this.storage.json('feeds.json'),
       this.storage.json('timezones.json'),
       this.storage.json('guides.json'),
-      this.storage.json('streams.json')
+      this.storage.json('streams.json'),
+      this.storage.json('logos.json')
     ])
 
     return {
@@ -75,7 +77,8 @@ export class DataLoader {
       feeds,
       timezones,
       guides,
-      streams
+      streams,
+      logos
     }
   }
 

scripts/core/dataProcessor.ts

@@ -1,6 +1,6 @@
+import { Channel, Feed, GuideChannel, Logo, Stream } from '../models'
 import { DataLoaderData } from '../types/dataLoader'
 import { Collection } from '@freearhey/core'
-import { Channel, Feed, Guide, Stream } from '../models'
 
 export class DataProcessor {
   constructor() {}
@@ -9,31 +9,48 @@ export class DataProcessor {
     let channels = new Collection(data.channels).map(data => new Channel(data))
     const channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
 
-    const guides = new Collection(data.guides).map(data => new Guide(data))
+    const guideChannels = new Collection(data.guides).map(data => new GuideChannel(data))
-    const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId())
+    const guideChannelsGroupedByStreamId = guideChannels.groupBy((channel: GuideChannel) =>
+      channel.getStreamId()
+    )
 
     const streams = new Collection(data.streams).map(data => new Stream(data))
     const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
 
-    const feeds = new Collection(data.feeds).map(data =>
+    let feeds = new Collection(data.feeds).map(data =>
       new Feed(data)
-        .withGuides(guidesGroupedByStreamId)
+        .withGuideChannels(guideChannelsGroupedByStreamId)
         .withStreams(streamsGroupedById)
         .withChannel(channelsKeyById)
     )
+    const feedsKeyByStreamId = feeds.keyBy((feed: Feed) => feed.getStreamId())
+
+    const logos = new Collection(data.logos).map(data =>
+      new Logo(data).withFeed(feedsKeyByStreamId)
+    )
+    const logosGroupedByChannelId = logos.groupBy((logo: Logo) => logo.channelId)
+    const logosGroupedByStreamId = logos.groupBy((logo: Logo) => logo.getStreamId())
+
+    feeds = feeds.map((feed: Feed) => feed.withLogos(logosGroupedByStreamId))
     const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
 
-    channels = channels.map((channel: Channel) => channel.withFeeds(feedsGroupedByChannelId))
+    channels = channels.map((channel: Channel) =>
+      channel.withFeeds(feedsGroupedByChannelId).withLogos(logosGroupedByChannelId)
+    )
 
     return {
+      guideChannelsGroupedByStreamId,
       feedsGroupedByChannelId,
-      guidesGroupedByStreamId,
+      logosGroupedByChannelId,
+      logosGroupedByStreamId,
       streamsGroupedById,
+      feedsKeyByStreamId,
       channelsKeyById,
+      guideChannels,
       channels,
       streams,
-      guides,
+      feeds,
-      feeds
+      logos
     }
   }
 }

scripts/core/grabber.ts

@@ -70,6 +70,10 @@ export class Grabber {
               }
             }
 
+            if (this.options.curl === true) {
+              config.curl = true
+            }
+
             const _programs = await this.grabber.grab(
               channel,
               date,

scripts/core/guide.ts

@@ -1,60 +0,0 @@
-import { Collection, Logger, DateTime, Storage, Zip } from '@freearhey/core'
-import { Channel } from 'epg-grabber'
-import { XMLTV } from '../core'
-import path from 'path'
-
-type GuideProps = {
-  channels: Collection
-  programs: Collection
-  logger: Logger
-  filepath: string
-  gzip: boolean
-}
-
-export class Guide {
-  channels: Collection
-  programs: Collection
-  logger: Logger
-  storage: Storage
-  filepath: string
-  gzip: boolean
-
-  constructor({ channels, programs, logger, filepath, gzip }: GuideProps) {
-    this.channels = channels
-    this.programs = programs
-    this.logger = logger
-    this.storage = new Storage(path.dirname(filepath))
-    this.filepath = filepath
-    this.gzip = gzip || false
-  }
-
-  async save() {
-    const channels = this.channels.uniqBy(
-      (channel: Channel) => `${channel.xmltv_id}:${channel.site}`
-    )
-    const programs = this.programs
-
-    const currDate = new DateTime(process.env.CURR_DATE || new Date().toISOString(), {
-      timezone: 'UTC'
-    })
-    const xmltv = new XMLTV({
-      channels,
-      programs,
-      date: currDate
-    })
-
-    const xmlFilepath = this.filepath
-    const xmlFilename = path.basename(xmlFilepath)
-    this.logger.info(`  saving to "${xmlFilepath}"...`)
-    await this.storage.save(xmlFilename, xmltv.toString())
-
-    if (this.gzip) {
-      const zip = new Zip()
-      const compressed = zip.compress(xmltv.toString())
-      const gzFilepath = `${this.filepath}.gz`
-      const gzFilename = path.basename(gzFilepath)
-      this.logger.info(`  saving to "${gzFilepath}"...`)
-      await this.storage.save(gzFilename, compressed)
-    }
-  }
-}

scripts/core/guideManager.ts

@@ -1,7 +1,12 @@
-import { Collection, Logger, Storage, StringTemplate } from '@freearhey/core'
+import { Collection, Logger, Zip, Storage, StringTemplate } from '@freearhey/core'
+import epgGrabber from 'epg-grabber'
 import { OptionValues } from 'commander'
-import { Channel, Program } from 'epg-grabber'
+import { Channel, Feed, Guide } from '../models'
-import { Guide } from '.'
+import path from 'path'
+import { DataLoader, DataProcessor } from '.'
+import { DataLoaderData } from '../types/dataLoader'
+import { DataProcessorData } from '../types/dataProcessor'
+import { DATA_DIR } from '../constants'
 
 type GuideManagerProps = {
   options: OptionValues
@@ -12,7 +17,6 @@ type GuideManagerProps = {
 
 export class GuideManager {
   options: OptionValues
-  storage: Storage
   logger: Logger
   channels: Collection
   programs: Collection
@@ -22,22 +26,51 @@ export class GuideManager {
     this.logger = logger
     this.channels = channels
     this.programs = programs
-    this.storage = new Storage()
   }
 
   async createGuides() {
     const pathTemplate = new StringTemplate(this.options.output)
 
+    const processor = new DataProcessor()
+    const dataStorage = new Storage(DATA_DIR)
+    const loader = new DataLoader({ storage: dataStorage })
+    const data: DataLoaderData = await loader.load()
+    const { feedsKeyByStreamId, channelsKeyById }: DataProcessorData = processor.process(data)
+
     const groupedChannels = this.channels
-      .orderBy([(channel: Channel) => channel.xmltv_id])
+      .map((channel: epgGrabber.Channel) => {
-      .uniqBy((channel: Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`)
+        if (channel.xmltv_id && !channel.icon) {
-      .groupBy((channel: Channel) => {
+          const foundFeed: Feed = feedsKeyByStreamId.get(channel.xmltv_id)
+          if (foundFeed && foundFeed.hasLogo()) {
+            channel.icon = foundFeed.getLogoUrl()
+          } else {
+            const [channelId] = channel.xmltv_id.split('@')
+            const foundChannel: Channel = channelsKeyById.get(channelId)
+            if (foundChannel && foundChannel.hasLogo()) {
+              channel.icon = foundChannel.getLogoUrl()
+            }
+          }
+        }
+
+        return channel
+      })
+      .orderBy([
+        (channel: epgGrabber.Channel) => channel.index,
+        (channel: epgGrabber.Channel) => channel.xmltv_id
+      ])
+      .uniqBy(
+        (channel: epgGrabber.Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`
+      )
+      .groupBy((channel: epgGrabber.Channel) => {
         return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' })
       })
 
     const groupedPrograms = this.programs
-      .orderBy([(program: Program) => program.channel, (program: Program) => program.start])
+      .orderBy([
-      .groupBy((program: Program) => {
+        (program: epgGrabber.Program) => program.channel,
+        (program: epgGrabber.Program) => program.start
+      ])
+      .groupBy((program: epgGrabber.Program) => {
         const lang =
           program.titles && program.titles.length && program.titles[0].lang
             ? program.titles[0].lang
@@ -51,11 +84,28 @@ export class GuideManager {
         filepath: groupKey,
         gzip: this.options.gzip,
         channels: new Collection(groupedChannels.get(groupKey)),
-        programs: new Collection(groupedPrograms.get(groupKey)),
+        programs: new Collection(groupedPrograms.get(groupKey))
-        logger: this.logger
       })
 
-      await guide.save()
+      await this.save(guide)
+    }
+  }
+
+  async save(guide: Guide) {
+    const storage = new Storage(path.dirname(guide.filepath))
+    const xmlFilepath = guide.filepath
+    const xmlFilename = path.basename(xmlFilepath)
+    this.logger.info(`  saving to "${xmlFilepath}"...`)
+    const xmltv = guide.toString()
+    await storage.save(xmlFilename, xmltv)
+
+    if (guide.gzip) {
+      const zip = new Zip()
+      const compressed = zip.compress(xmltv)
+      const gzFilepath = `${guide.filepath}.gz`
+      const gzFilename = path.basename(gzFilepath)
+      this.logger.info(`  saving to "${gzFilepath}"...`)
+      await storage.save(gzFilename, compressed)
     }
   }
 }

scripts/core/index.ts

@@ -4,7 +4,6 @@ export * from './configLoader'
 export * from './dataLoader'
 export * from './dataProcessor'
 export * from './grabber'
-export * from './guide'
 export * from './guideManager'
 export * from './htmlTable'
 export * from './issueLoader'
@@ -13,5 +12,3 @@ export * from './job'
 export * from './proxyParser'
 export * from './queue'
 export * from './queueCreator'
-export * from './xml'
-export * from './xmltv'

scripts/core/issueLoader.ts

@@ -23,6 +23,7 @@ export class IssueLoader {
         repo: REPO,
         per_page: 100,
         labels,
+        state: 'open',
         headers: {
           'X-GitHub-Api-Version': '2022-11-28'
         }

scripts/core/proxyParser.ts

@@ -2,9 +2,9 @@ import { URL } from 'node:url'
 
 type ProxyParserResult = {
   protocol: string | null
-  auth: {
+  auth?: {
-    username: string | null
+    username?: string
-    password: string | null
+    password?: string
   }
   host: string
   port: number | null
@@ -14,14 +14,18 @@ export class ProxyParser {
   parse(_url: string): ProxyParserResult {
     const parsed = new URL(_url)
 
-    return {
+    const result: ProxyParserResult = {
       protocol: parsed.protocol.replace(':', '') || null,
-      auth: {
-        username: parsed.username || null,
-        password: parsed.password || null
-      },
       host: parsed.hostname,
       port: parsed.port ? parseInt(parsed.port) : null
     }
+
+    if (parsed.username || parsed.password) {
+      result.auth = {}
+      if (parsed.username) result.auth.username = parsed.username
+      if (parsed.password) result.auth.password = parsed.password
+    }
+
+    return result
   }
 }

scripts/core/queueCreator.ts

@@ -1,15 +1,14 @@
 import { Storage, Collection, DateTime, Logger } from '@freearhey/core'
-import { ChannelsParser, ConfigLoader, Queue } from './'
 import { SITES_DIR, DATA_DIR } from '../constants'
+import { GrabOptions } from '../commands/epg/grab'
+import { ConfigLoader, Queue } from './'
 import { SiteConfig } from 'epg-grabber'
 import path from 'path'
-import { GrabOptions } from '../commands/epg/grab'
-import { Channel } from '../models'
 
 type QueueCreatorProps = {
   logger: Logger
   options: GrabOptions
-  parsedChannels: Collection
+  channels: Collection
 }
 
 export class QueueCreator {
@@ -17,42 +16,29 @@ export class QueueCreator {
   logger: Logger
   sitesStorage: Storage
   dataStorage: Storage
-  parser: ChannelsParser
+  channels: Collection
-  parsedChannels: Collection
   options: GrabOptions
 
-  constructor({ parsedChannels, logger, options }: QueueCreatorProps) {
+  constructor({ channels, logger, options }: QueueCreatorProps) {
-    this.parsedChannels = parsedChannels
+    this.channels = channels
     this.logger = logger
     this.sitesStorage = new Storage()
     this.dataStorage = new Storage(DATA_DIR)
-    this.parser = new ChannelsParser({ storage: new Storage() })
     this.options = options
     this.configLoader = new ConfigLoader()
   }
 
   async create(): Promise<Queue> {
-    const channelsContent = await this.dataStorage.json('channels.json')
+    let index = 0
-    const channels = new Collection(channelsContent).map(data => new Channel(data))
-
     const queue = new Queue()
-    for (const channel of this.parsedChannels.all()) {
+    for (const channel of this.channels.all()) {
+      channel.index = index++
       if (!channel.site || !channel.site_id || !channel.name) continue
 
       const configPath = path.resolve(SITES_DIR, `${channel.site}/${channel.site}.config.js`)
       const config: SiteConfig = await this.configLoader.load(configPath)
 
-      if (channel.xmltv_id) {
+      if (!channel.xmltv_id) {
-        if (!channel.icon) {
-          const found: Channel = channels.first(
-            (_channel: Channel) => _channel.id === channel.xmltv_id
-          )
-
-          if (found) {
-            channel.icon = found.logo
-          }
-        }
-      } else {
         channel.xmltv_id = channel.site_id
       }
 

scripts/core/xml.ts

@@ -1,56 +0,0 @@
-import { Collection } from '@freearhey/core'
-import { Channel } from 'epg-grabber'
-
-export class XML {
-  items: Collection
-
-  constructor(items: Collection) {
-    this.items = items
-  }
-
-  toString() {
-    let output = '<?xml version="1.0" encoding="UTF-8"?>\r\n<channels>\r\n'
-
-    this.items.forEach((channel: Channel) => {
-      const logo = channel.logo ? ` logo="${channel.logo}"` : ''
-      const xmltv_id = channel.xmltv_id || ''
-      const lang = channel.lang || ''
-      const site_id = channel.site_id || ''
-      output += `  <channel site="${channel.site}" lang="${lang}" xmltv_id="${escapeString(
-        xmltv_id
-      )}" site_id="${site_id}"${logo}>${escapeString(channel.name)}</channel>\r\n`
-    })
-
-    output += '</channels>\r\n'
-
-    return output
-  }
-}
-
-function escapeString(value: string, defaultValue: string = '') {
-  if (!value) return defaultValue
-
-  const regex = new RegExp(
-    '((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
-      'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
-      'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
-      '|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
-      'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
-      '[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
-      'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
-      '(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
-    'g'
-  )
-
-  value = String(value || '').replace(regex, '')
-
-  return value
-    .replace(/&/g, '&amp;')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-    .replace(/'/g, '&apos;')
-    .replace(/\n|\r/g, ' ')
-    .replace(/  +/g, ' ')
-    .trim()
-}

scripts/core/xmltv.ts

@@ -1,28 +0,0 @@
-import { DateTime, Collection } from '@freearhey/core'
-import { generateXMLTV } from 'epg-grabber'
-
-type XMLTVProps = {
-  channels: Collection
-  programs: Collection
-  date: DateTime
-}
-
-export class XMLTV {
-  channels: Collection
-  programs: Collection
-  date: DateTime
-
-  constructor({ channels, programs, date }: XMLTVProps) {
-    this.channels = channels
-    this.programs = programs
-    this.date = date
-  }
-
-  toString() {
-    return generateXMLTV({
-      channels: this.channels.all(),
-      programs: this.programs.all(),
-      date: this.date.toJSON()
-    })
-  }
-}

scripts/models/channel.ts

@@ -1,26 +1,28 @@
 import { ChannelData, ChannelSearchableData } from '../types/channel'
 import { Collection, Dictionary } from '@freearhey/core'
-import { Stream, Guide, Feed } from './'
+import { Stream, Feed, Logo, GuideChannel } from './'
 
 export class Channel {
-  id: string
+  id?: string
-  name: string
+  name?: string
   altNames?: Collection
   network?: string
   owners?: Collection
-  countryCode: string
+  countryCode?: string
   subdivisionCode?: string
   cityName?: string
   categoryIds?: Collection
-  isNSFW: boolean
+  isNSFW: boolean = false
   launched?: string
   closed?: string
   replacedBy?: string
   website?: string
-  logo?: string
   feeds?: Collection
+  logos: Collection = new Collection()
+
+  constructor(data?: ChannelData) {
+    if (!data) return
 
-  constructor(data: ChannelData) {
     this.id = data.id
     this.name = data.name
     this.altNames = new Collection(data.alt_names)
@@ -35,11 +37,16 @@ export class Channel {
     this.closed = data.closed || undefined
     this.replacedBy = data.replaced_by || undefined
     this.website = data.website || undefined
-    this.logo = data.logo
   }
 
   withFeeds(feedsGroupedByChannelId: Dictionary): this {
-    this.feeds = new Collection(feedsGroupedByChannelId.get(this.id))
+    if (this.id) this.feeds = new Collection(feedsGroupedByChannelId.get(this.id))
+
+    return this
+  }
+
+  withLogos(logosGroupedByChannelId: Dictionary): this {
+    if (this.id) this.logos = new Collection(logosGroupedByChannelId.get(this.id))
 
     return this
   }
@@ -50,19 +57,19 @@ export class Channel {
     return this.feeds
   }
 
-  getGuides(): Collection {
+  getGuideChannels(): Collection {
-    let guides = new Collection()
+    let channels = new Collection()
 
     this.getFeeds().forEach((feed: Feed) => {
-      guides = guides.concat(feed.getGuides())
+      channels = channels.concat(feed.getGuideChannels())
     })
 
-    return guides
+    return channels
   }
 
-  getGuideNames(): Collection {
+  getGuideChannelNames(): Collection {
-    return this.getGuides()
+    return this.getGuideChannels()
-      .map((guide: Guide) => guide.siteName)
+      .map((channel: GuideChannel) => channel.siteName)
       .uniq()
   }
 
@@ -100,12 +107,56 @@ export class Channel {
     return this.altNames || new Collection()
   }
 
+  getLogos(): Collection {
+    function feed(logo: Logo): number {
+      if (!logo.feed) return 1
+      if (logo.feed.isMain) return 1
+
+      return 0
+    }
+
+    function format(logo: Logo): number {
+      const levelByFormat: { [key: string]: number } = {
+        SVG: 0,
+        PNG: 3,
+        APNG: 1,
+        WebP: 1,
+        AVIF: 1,
+        JPEG: 2,
+        GIF: 1
+      }
+
+      return logo.format ? levelByFormat[logo.format] : 0
+    }
+
+    function size(logo: Logo): number {
+      return Math.abs(512 - logo.width) + Math.abs(512 - logo.height)
+    }
+
+    return this.logos.orderBy([feed, format, size], ['desc', 'desc', 'asc'], false)
+  }
+
+  getLogo(): Logo | undefined {
+    return this.getLogos().first()
+  }
+
+  hasLogo(): boolean {
+    return this.getLogos().notEmpty()
+  }
+
+  getLogoUrl(): string {
+    const logo = this.getLogo()
+    if (!logo) return ''
+
+    return logo.url || ''
+  }
+
   getSearchable(): ChannelSearchableData {
     return {
       id: this.getId(),
       name: this.getName(),
       altNames: this.getAltNames().all(),
-      guideNames: this.getGuideNames().all(),
+      guideNames: this.getGuideChannelNames().all(),
       streamNames: this.getStreamNames().all(),
       feedFullNames: this.getFeedFullNames().all()
     }

scripts/models/channelList.ts

@@ -0,0 +1,77 @@
+import { Collection } from '@freearhey/core'
+import epgGrabber from 'epg-grabber'
+
+export class ChannelList {
+  channels: Collection = new Collection()
+
+  constructor(data: { channels: epgGrabber.Channel[] }) {
+    this.channels = new Collection(data.channels)
+  }
+
+  add(channel: epgGrabber.Channel): this {
+    this.channels.add(channel)
+
+    return this
+  }
+
+  get(siteId: string): epgGrabber.Channel | undefined {
+    return this.channels.find((channel: epgGrabber.Channel) => channel.site_id == siteId)
+  }
+
+  sort(): this {
+    this.channels = this.channels.orderBy([
+      (channel: epgGrabber.Channel) => channel.lang || '_',
+      (channel: epgGrabber.Channel) => (channel.xmltv_id ? channel.xmltv_id.toLowerCase() : '0'),
+      (channel: epgGrabber.Channel) => channel.site_id
+    ])
+
+    return this
+  }
+
+  toString() {
+    function escapeString(value: string, defaultValue: string = '') {
+      if (!value) return defaultValue
+
+      const regex = new RegExp(
+        '((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
+          'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
+          'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
+          '|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
+          'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
+          '[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
+          'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
+          '(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
+        'g'
+      )
+
+      value = String(value || '').replace(regex, '')
+
+      return value
+        .replace(/&/g, '&')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;')
+        .replace(/\n|\r/g, ' ')
+        .replace(/  +/g, ' ')
+        .trim()
+    }
+
+    let output = '<?xml version="1.0" encoding="UTF-8"?>\r\n<channels>\r\n'
+
+    this.channels.forEach((channel: epgGrabber.Channel) => {
+      const logo = channel.logo ? ` logo="${channel.logo}"` : ''
+      const xmltv_id = channel.xmltv_id ? escapeString(channel.xmltv_id) : ''
+      const lang = channel.lang || ''
+      const site_id = channel.site_id || ''
+      const site = channel.site || ''
+      const displayName = channel.name ? escapeString(channel.name) : ''
+
+      output += `  <channel site="${site}" lang="${lang}" xmltv_id="${xmltv_id}" site_id="${site_id}"${logo}>${displayName}</channel>\r\n`
+    })
+
+    output += '</channels>\r\n'
+
+    return output
+  }
+}

scripts/models/feed.ts

@@ -1,6 +1,6 @@
 import { Collection, Dictionary } from '@freearhey/core'
 import { FeedData } from '../types/feed'
-import { Channel } from './channel'
+import { Logo, Channel } from '.'
 
 export class Feed {
   channelId: string
@@ -12,8 +12,9 @@ export class Feed {
   languageCodes: Collection
   timezoneIds: Collection
   videoFormat: string
-  guides?: Collection
+  guideChannels?: Collection
   streams?: Collection
+  logos: Collection = new Collection()
 
   constructor(data: FeedData) {
     this.channelId = data.channel
@@ -42,20 +43,30 @@ export class Feed {
     return this
   }
 
-  withGuides(guidesGroupedByStreamId: Dictionary): this {
+  withGuideChannels(guideChannelsGroupedByStreamId: Dictionary): this {
-    this.guides = new Collection(guidesGroupedByStreamId.get(`${this.channelId}@${this.id}`))
+    this.guideChannels = new Collection(
+      guideChannelsGroupedByStreamId.get(`${this.channelId}@${this.id}`)
+    )
 
     if (this.isMain) {
-      this.guides = this.guides.concat(new Collection(guidesGroupedByStreamId.get(this.channelId)))
+      this.guideChannels = this.guideChannels.concat(
+        new Collection(guideChannelsGroupedByStreamId.get(this.channelId))
+      )
     }
 
     return this
   }
 
-  getGuides(): Collection {
+  withLogos(logosGroupedByStreamId: Dictionary): this {
-    if (!this.guides) return new Collection()
+    this.logos = new Collection(logosGroupedByStreamId.get(this.getStreamId()))
 
-    return this.guides
+    return this
+  }
+
+  getGuideChannels(): Collection {
+    if (!this.guideChannels) return new Collection()
+
+    return this.guideChannels
   }
 
   getStreams(): Collection {
@@ -69,4 +80,45 @@ export class Feed {
 
     return `${this.channel.name} ${this.name}`
   }
+
+  getStreamId(): string {
+    return `${this.channelId}@${this.id}`
+  }
+
+  getLogos(): Collection {
+    function format(logo: Logo): number {
+      const levelByFormat: { [key: string]: number } = {
+        SVG: 0,
+        PNG: 3,
+        APNG: 1,
+        WebP: 1,
+        AVIF: 1,
+        JPEG: 2,
+        GIF: 1
+      }
+
+      return logo.format ? levelByFormat[logo.format] : 0
+    }
+
+    function size(logo: Logo): number {
+      return Math.abs(512 - logo.width) + Math.abs(512 - logo.height)
+    }
+
+    return this.logos.orderBy([format, size], ['desc', 'asc'], false)
+  }
+
+  getLogo(): Logo | undefined {
+    return this.getLogos().first()
+  }
+
+  hasLogo(): boolean {
+    return this.getLogos().notEmpty()
+  }
+
+  getLogoUrl(): string {
+    const logo = this.getLogo()
+    if (!logo) return ''
+
+    return logo.url || ''
+  }
 }

scripts/models/guide.ts

@@ -1,35 +1,35 @@
-import type { GuideData } from '../types/guide'
+import { Collection, DateTime } from '@freearhey/core'
-import { uniqueId } from 'lodash'
+import { generateXMLTV } from 'epg-grabber'
+
+type GuideData = {
+  channels: Collection
+  programs: Collection
+  filepath: string
+  gzip: boolean
+}
 
 export class Guide {
-  channelId?: string
+  channels: Collection
-  feedId?: string
+  programs: Collection
-  siteDomain?: string
+  filepath: string
-  siteId?: string
+  gzip: boolean
-  siteName?: string
-  languageCode?: string
 
-  constructor(data?: GuideData) {
+  constructor({ channels, programs, filepath, gzip }: GuideData) {
-    if (!data) return
+    this.channels = channels
-
+    this.programs = programs
-    this.channelId = data.channel
+    this.filepath = filepath
-    this.feedId = data.feed
+    this.gzip = gzip || false
-    this.siteDomain = data.site
-    this.siteId = data.site_id
-    this.siteName = data.site_name
-    this.languageCode = data.lang
   }
 
-  getUUID(): string {
+  toString() {
-    if (!this.getStreamId() || !this.siteId) return uniqueId()
+    const currDate = new DateTime(process.env.CURR_DATE || new Date().toISOString(), {
+      timezone: 'UTC'
+    })
 
-    return this.getStreamId() + this.siteId
+    return generateXMLTV({
-  }
+      channels: this.channels.all(),
-
+      programs: this.programs.all(),
-  getStreamId(): string | undefined {
+      date: currDate.toJSON()
-    if (!this.channelId) return undefined
+    })
-    if (!this.feedId) return this.channelId
-
-    return `${this.channelId}@${this.feedId}`
   }
 }

scripts/models/guideChannel.ts

@@ -0,0 +1,59 @@
+import { Dictionary } from '@freearhey/core'
+import epgGrabber from 'epg-grabber'
+import { Feed, Channel } from '.'
+
+export class GuideChannel {
+  channelId?: string
+  channel?: Channel
+  feedId?: string
+  feed?: Feed
+  xmltvId?: string
+  languageCode?: string
+  siteId?: string
+  logoUrl?: string
+  siteDomain?: string
+  siteName?: string
+
+  constructor(data: epgGrabber.Channel) {
+    const [channelId, feedId] = data.xmltv_id ? data.xmltv_id.split('@') : [undefined, undefined]
+
+    this.channelId = channelId
+    this.feedId = feedId
+    this.xmltvId = data.xmltv_id
+    this.languageCode = data.lang
+    this.siteId = data.site_id
+    this.logoUrl = data.logo
+    this.siteDomain = data.site
+    this.siteName = data.name
+  }
+
+  withChannel(channelsKeyById: Dictionary): this {
+    if (this.channelId) this.channel = channelsKeyById.get(this.channelId)
+
+    return this
+  }
+
+  withFeed(feedsKeyByStreamId: Dictionary): this {
+    if (this.feedId) this.feed = feedsKeyByStreamId.get(this.getStreamId())
+
+    return this
+  }
+
+  getStreamId(): string {
+    if (!this.channelId) return ''
+    if (!this.feedId) return this.channelId
+
+    return `${this.channelId}@${this.feedId}`
+  }
+
+  toJSON() {
+    return {
+      channel: this.channelId || null,
+      feed: this.feedId || null,
+      site: this.siteDomain || '',
+      site_id: this.siteId || '',
+      site_name: this.siteName || '',
+      lang: this.languageCode || ''
+    }
+  }
+}

scripts/models/index.ts

@@ -1,6 +1,9 @@
-export * from './issue'
-export * from './site'
 export * from './channel'
 export * from './feed'
-export * from './stream'
 export * from './guide'
+export * from './guideChannel'
+export * from './issue'
+export * from './logo'
+export * from './site'
+export * from './stream'
+export * from './channelList'

scripts/models/logo.ts

@@ -0,0 +1,41 @@
+import { Collection, type Dictionary } from '@freearhey/core'
+import type { LogoData } from '../types/logo'
+import { type Feed } from './feed'
+
+export class Logo {
+  channelId?: string
+  feedId?: string
+  feed?: Feed
+  tags: Collection = new Collection()
+  width: number = 0
+  height: number = 0
+  format?: string
+  url?: string
+
+  constructor(data?: LogoData) {
+    if (!data) return
+
+    this.channelId = data.channel
+    this.feedId = data.feed || undefined
+    this.tags = new Collection(data.tags)
+    this.width = data.width
+    this.height = data.height
+    this.format = data.format || undefined
+    this.url = data.url
+  }
+
+  withFeed(feedsKeyByStreamId: Dictionary): this {
+    if (!this.feedId) return this
+
+    this.feed = feedsKeyByStreamId.get(this.getStreamId())
+
+    return this
+  }
+
+  getStreamId(): string {
+    if (!this.channelId) return ''
+    if (!this.feedId) return this.channelId
+
+    return `${this.channelId}@${this.feedId}`
+  }
+}

scripts/types/channel.d.ts

@@ -15,7 +15,6 @@ export type ChannelData = {
   closed: string
   replaced_by: string
   website: string
-  logo: string
 }
 
 export type ChannelSearchableData = {

scripts/types/dataLoader.d.ts

@@ -16,4 +16,5 @@ export type DataLoaderData = {
   timezones: object | object[]
   guides: object | object[]
   streams: object | object[]
+  logos: object | object[]
 }

scripts/types/dataProcessor.d.ts

@@ -1,12 +1,16 @@
 import { Collection, Dictionary } from '@freearhey/core'
 
 export type DataProcessorData = {
+  guideChannelsGroupedByStreamId: Dictionary
   feedsGroupedByChannelId: Dictionary
-  guidesGroupedByStreamId: Dictionary
+  logosGroupedByChannelId: Dictionary
+  logosGroupedByStreamId: Dictionary
+  feedsKeyByStreamId: Dictionary
   streamsGroupedById: Dictionary
   channelsKeyById: Dictionary
+  guideChannels: Collection
   channels: Collection
   streams: Collection
-  guides: Collection
   feeds: Collection
+  logos: Collection
 }

scripts/types/logo.d.ts

@@ -0,0 +1,9 @@
+export type LogoData = {
+  channel: string
+  feed: string | null
+  tags: string[]
+  width: number
+  height: number
+  format: string | null
+  url: string
+}

tests/__data__/expected/epg_grab/base.guide.xml

@@ -1,8 +1,9 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
+<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
 <channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
-<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url></channel>
-<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
-<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
+<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 </tv>

tests/__data__/expected/epg_grab/custom_channels.guide.xml

@@ -1,14 +1,15 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
 <channel id="Channel1.us"><display-name>Custom Channel 1</display-name><url>https://example.com</url></channel>
-<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example2.com</url></channel>
 <channel id="Channel2.us"><display-name>Custom Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url></channel>
-<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example2.com</url></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
-<channel id="Channel4.us"><display-name>Channel 4</display-name><url>https://example2.com</url></channel>
+<channel id="Channel3.us@Wrong"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example2.com</url></channel>
-<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
+<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/><url>https://example2.com</url></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example2.com</url></channel>
 <programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221019044000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example2.com)</title></programme>
-<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
-<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="en">Program1 (example2.com)</title></programme>
+<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
-<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel4.us"><title lang="en">Program1 (example2.com)</title></programme>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel3.us@Wrong"><title lang="en">Program1 (example2.com)</title></programme>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel4.us@HD"><title lang="en">Program1 (example2.com)</title></programme>
 </tv>

tests/__data__/expected/epg_grab/guides/en/example.com.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
+<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
 <channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
-<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url></channel>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
-<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
+<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
-<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 </tv>

tests/__data__/expected/epg_grab/lang.guide.xml

@@ -3,6 +3,6 @@
 <channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example.com</url></channel>
 <programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
 <programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
-<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
-<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
+<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
 </tv>

tests/__data__/expected/epg_grab/proxy.guide.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
+<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
+<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
+<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
+<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
+</tv>

tests/__data__/expected/epg_grab/template.guide.xml

@@ -1,14 +1,15 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
+<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
 <channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
-<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example2.com</url></channel>
-<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url></channel>
 <channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example2.com</url></channel>
-<channel id="Channel4.us"><display-name>Channel 4</display-name><url>https://example2.com</url></channel>
+<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/><url>https://example2.com</url></channel>
-<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
+<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example2.com</url></channel>
 <programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221019044000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example2.com)</title></programme>
-<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
 <programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
-<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="en">Program1 (example2.com)</title></programme>
+<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
-<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel4.us"><title lang="en">Program1 (example2.com)</title></programme>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="en">Program1 (example2.com)</title></programme>
+<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel4.us@HD"><title lang="en">Program1 (example2.com)</title></programme>
 </tv>

tests/__data__/input/__data__/channels.json

@@ -9,8 +9,7 @@
     "categories": [],
     "is_nsfw": false,
     "closed": "2020-01-01",
-    "replaced_by": "R6.co",
+    "replaced_by": "R6.co"
-    "logo": "https://www.directv.com/images/logos/channels/dark/large/579.png"
   },
   {
     "id": "Bravos.us",
@@ -20,8 +19,7 @@
     "subdivision": null,
     "city": null,
     "categories": [],
-    "is_nsfw": false,
+    "is_nsfw": false
-    "logo": "https://www.directv.com/images/logos/channels/dark/large/579.png"
   },
   {
     "id": "CNNInternational.us",
@@ -34,8 +32,7 @@
     "categories": [
       "news"
     ],
-    "is_nsfw": false,
+    "is_nsfw": false
-    "logo": "https://i.imgur.com/2BXCg0x.jpg"
   },
   {
     "id": "MNetMovies2.za",
@@ -45,11 +42,10 @@
     "subdivision": null,
     "city": null,
     "categories": [],
-    "is_nsfw": false,
+    "is_nsfw": false
-    "logo": "https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png"
   },
-  {"id":"6eren.dk","name":"6'eren","alt_names":[],"network":null,"owners":["Warner Bros. Discovery EMEA"],"country":"DK","subdivision":null,"city":null,"broadcast_area":["c/DK"],"languages":["dan"],"categories":[],"is_nsfw":false,"launched":"2009-01-01","closed":null,"replaced_by":null,"website":"http://www.6-eren.dk/","logo":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"},
+  {"id":"6eren.dk","name":"6'eren","alt_names":[],"network":null,"owners":["Warner Bros. Discovery EMEA"],"country":"DK","subdivision":null,"city":null,"broadcast_area":["c/DK"],"languages":["dan"],"categories":[],"is_nsfw":false,"launched":"2009-01-01","closed":null,"replaced_by":null,"website":"http://www.6-eren.dk/"},
-  {"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/","logo":"https://i.imgur.com/rPzH88J.png"},
+  {"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/"},
   {
     "id": "CNN.us",
     "name": "CNN",
@@ -58,9 +54,8 @@
     "subdivision": null,
     "city": null,
     "categories": [],
-    "is_nsfw": false,
+    "is_nsfw": false
-    "logo": "https://www.directv.com/images/logos/channels/dark/large/579.png"
   },
-  {"id":"Channel2.us","name":"Channel 2 [API]","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/US"],"languages":["eng"],"categories":[],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"","logo":"https://i.imgur.com/rPzH88J.png"},
+  {"id":"Channel2.us","name":"Channel 2 [API]","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/US"],"languages":["eng"],"categories":[],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":""},
-  {"id":"Channel3.us","name":"Channel 3 [API]","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/US"],"languages":["eng"],"categories":[],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"","logo":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"}
+  {"id":"Channel3.us","name":"Channel 3 [API]","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/US"],"languages":["eng"],"categories":[],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":""}
 ]

tests/__data__/input/__data__/feeds.json

@@ -26,5 +26,33 @@
     ],
     "languages": [],
     "video_format": "480i"
+  },
+  {
+    "channel": "Bravo.us",
+    "id": "East",
+    "name": "East",
+    "is_main": true,
+    "broadcast_area": [
+      "r/EUR"
+    ],
+    "timezones": [
+      "America/New_York"
+    ],
+    "languages": [],
+    "video_format": "480i"
+  },
+  {
+    "channel": "Channel4.us",
+    "id": "HD",
+    "name": "HD",
+    "is_main": true,
+    "broadcast_area": [
+      "r/EUR"
+    ],
+    "timezones": [
+      "America/New_York"
+    ],
+    "languages": [],
+    "video_format": "480i"
   }
 ]

tests/__data__/input/__data__/logos.json

@@ -0,0 +1,4 @@
+[
+  {"channel":"Channel3.us","feed":null,"tags":[],"width":334,"height":210,"format":"PNG","url":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"},
+  {"channel":"Channel4.us","feed":"HD","tags":[],"width":334,"height":210,"format":"PNG","url":"https://i.imgur.com/BPzH88J.png"}
+]

tests/__data__/input/channels_lint/valid.channels.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <channels>
-  <channel site="example.com" lang="en" xmltv_id="Bravo.us@East" site_id="140#Tes't">Bravo's</channel>
+  <channel site="example.com" lang="en" xmltv_id="Bravo.us@East" site_id="140#Tes't" lcn="36">Bravo's</channel>
 </channels>

tests/__data__/input/channels_validate/wrong_channel_id.channels.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <channels site="example.com">
   <channel lang="en" xmltv_id="CNNInternational" site_id="140">CNN International</channel>
-  <channel lang="en" xmltv_id="Bravo.us@East" site_id="150">Bravo</channel>
+  <channel lang="en" xmltv_id="Bravo.us" site_id="150">Bravo</channel>
-  <channel lang="en" xmltv_id="Bravo.us@West" site_id="151">Bravo</channel>
+  <channel lang="en" xmltv_id="Bravo.us" site_id="151">Bravo</channel>
 </channels>

tests/__data__/input/channels_validate/wrong_feed_id.channels.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<channels site="example.com">
+  <channel lang="en" xmltv_id="Bravo.us@West" site_id="150">Bravo</channel>
+</channels>

tests/__data__/input/epg_grab/custom.channels.xml

@@ -3,7 +3,7 @@
   <channel site="example.com" lang="en" xmltv_id="Channel1.us" site_id="140">Custom Channel 1</channel>
   <channel site="example.com" lang="en" xmltv_id="Channel2.us" site_id="142" logo="https://i.imgur.com/qmRnD0M.png">Custom Channel 2</channel>
   <channel site="example.com" lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
-  <channel site="example2.com" lang="en" xmltv_id="Channel3.us" site_id="150">Channel 3</channel>
+  <channel site="example2.com" lang="en" xmltv_id="Channel3.us@Wrong" site_id="150">Channel 3</channel>
-  <channel site="example2.com" lang="en" xmltv_id="Channel4.us" site_id="152">Channel 4</channel>
+  <channel site="example2.com" lang="en" xmltv_id="Channel4.us@HD" site_id="152">Channel 4</channel>
   <channel site="example2.com" lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
 </channels>

tests/__data__/input/epg_grab/example.com/example.com.config.js

@@ -20,7 +20,7 @@ module.exports = {
     return [
       {
         title: 'Program1 (example.com)',
-        start: `${date.format('YYYY-MM-DD')}T04:30:00.000Z`,
+        start: `${date.format('YYYY-MM-DD')}T04:31:00.000Z`,
         stop: `${date.format('YYYY-MM-DD')}T07:10:00.000Z`
       }
     ]

tests/__data__/input/epg_grab/sites/example.com/example.com.channels.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <channels site="example.com">
+  <channel lang="en" xmltv_id="Channel2.us" site_id="142" logo="https://i.imgur.com/qmRnD0M.png" lcn="36">Channel 2</channel>
   <channel lang="en" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
-  <channel lang="en" xmltv_id="Channel2.us" site_id="142" logo="https://i.imgur.com/qmRnD0M.png">Channel 2</channel>
   <channel lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
 </channels>

tests/__data__/input/epg_grab/sites/example.com/example.com.config.js

@@ -20,7 +20,7 @@ module.exports = {
     return [
       {
         title: 'Program1 (example.com)',
-        start: `${date.format('YYYY-MM-DD')}T04:30:00.000Z`,
+        start: `${date.format('YYYY-MM-DD')}T04:31:00.000Z`,
         stop: `${date.format('YYYY-MM-DD')}T07:10:00.000Z`
       }
     ]

tests/__data__/input/epg_grab/sites/example2.com/example2.com.channels.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <channels site="example2.com">
   <channel lang="en" xmltv_id="Channel3.us" site_id="150">Channel 3</channel>
-  <channel lang="en" xmltv_id="Channel4.us" site_id="152">Channel 4</channel>
+  <channel lang="en" xmltv_id="Channel4.us@HD" site_id="152">Channel 4</channel>
   <channel lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
 </channels>

tests/__data__/input/epg_grab/sites/example2.com/example2.com.config.js

@@ -15,7 +15,7 @@ module.exports = {
     return [
       {
         title: 'Program1 (example2.com)',
-        start: `${date.format('YYYY-MM-DD')}T04:30:00.000Z`,
+        start: `${date.format('YYYY-MM-DD')}T04:31:00.000Z`,
         stop: `${date.format('YYYY-MM-DD')}T07:10:00.000Z`
       }
     ]

tests/commands/channels/validate.test.ts

@@ -28,20 +28,40 @@ describe('channels:validate', () => {
     }
   })
 
-  it('will show a message if the file contains a channel with wrong xmltv_id', () => {
+  it('will show a message if the file contains a channel with wrong channel id', () => {
     try {
-      const cmd = `${ENV_VAR} npm run channels:validate --- tests/__data__/input/channels_validate/wrong_xmltv_id.channels.xml`
+      const cmd = `${ENV_VAR} npm run channels:validate --- tests/__data__/input/channels_validate/wrong_channel_id.channels.xml`
       const stdout = execSync(cmd, { encoding: 'utf8' })
       if (process.env.DEBUG === 'true') console.log(cmd, stdout)
       process.exit(1)
     } catch (error) {
       expect((error as ExecError).status).toBe(1)
       expect((error as ExecError).stdout).toContain(`
-┌─────────┬──────────────────┬──────┬────────────────────┬─────────┬─────────────────────┐
+┌─────────┬────────────────────┬──────┬────────────────────┬─────────┬─────────────────────┐
 │ (index) │ type               │ lang │ xmltv_id           │ site_id │ name                │
-├─────────┼──────────────────┼──────┼────────────────────┼─────────┼─────────────────────┤
+├─────────┼────────────────────┼──────┼────────────────────┼─────────┼─────────────────────┤
-│ 0       │ 'wrong_xmltv_id' │ 'en' │ 'CNNInternational' │ '140'   │ 'CNN International' │
+│ 0       │ 'wrong_channel_id' │ 'en' │ 'CNNInternational' │ '140'   │ 'CNN International' │
-└─────────┴──────────────────┴──────┴────────────────────┴─────────┴─────────────────────┘
+└─────────┴────────────────────┴──────┴────────────────────┴─────────┴─────────────────────┘
+
+1 error(s) in 1 file(s)
+`)
+    }
+  })
+
+  it('will show a message if the file contains a channel with wrong feed id', () => {
+    try {
+      const cmd = `${ENV_VAR} npm run channels:validate --- tests/__data__/input/channels_validate/wrong_feed_id.channels.xml`
+      const stdout = execSync(cmd, { encoding: 'utf8' })
+      if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+      process.exit(1)
+    } catch (error) {
+      expect((error as ExecError).status).toBe(1)
+      expect((error as ExecError).stdout).toContain(`
+┌─────────┬─────────────────┬──────┬─────────────────┬─────────┬─────────┐
+│ (index) │ type            │ lang │ xmltv_id        │ site_id │ name    │
+├─────────┼─────────────────┼──────┼─────────────────┼─────────┼─────────┤
+│ 0       │ 'wrong_feed_id' │ 'en' │ 'Bravo.us@West' │ '150'   │ 'Bravo' │
+└─────────┴─────────────────┴──────┴─────────────────┴─────────┴─────────┘
 
 1 error(s) in 1 file(s)
 `)

tests/commands/epg/grab.test.ts

@@ -4,7 +4,8 @@ import { Zip } from '@freearhey/core'
 import fs from 'fs-extra'
 import path from 'path'
 
-const ENV_VAR = 'cross-env SITES_DIR=tests/__data__/input/epg_grab/sites CURR_DATE=2022-10-20 DATA_DIR=tests/__data__/input/__data__'
+const ENV_VAR =
+  'cross-env SITES_DIR=tests/__data__/input/epg_grab/sites CURR_DATE=2022-10-20 DATA_DIR=tests/__data__/input/__data__'
 
 beforeEach(() => {
   fs.emptyDirSync('tests/__data__/output')
@@ -14,85 +15,12 @@ describe('epg:grab', () => {
   it('can grab epg by site name', () => {
     const cmd = `${ENV_VAR} npm run grab --- --site=example.com --output="${path.resolve(
       'tests/__data__/output/guide.xml'
-    )}"`
+    )}" --timeout=100`
     const stdout = execSync(cmd, { encoding: 'utf8' })
     if (process.env.DEBUG === 'true') console.log(cmd, stdout)
 
     expect(content('tests/__data__/output/guide.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guide_2.xml')
+      content('tests/__data__/expected/epg_grab/base.guide.xml')
-    )
-  })
-
-  it('can grab epg with multiple channels.xml files', () => {
-    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/**/*.channels.xml --output=tests/__data__/output/guide.xml`
-    const stdout = execSync(cmd, { encoding: 'utf8' })
-    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
-
-    expect(content('tests/__data__/output/guide.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guide.xml')
-    )
-  })
-
-  it('can grab epg with gzip option enabled', async () => {
-    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/**/*.channels.xml --output="${path.resolve(
-      'tests/__data__/output/guide.xml'
-    )}" --gzip`
-    const stdout = execSync(cmd, { encoding: 'utf8' })
-    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
-
-    expect(content('tests/__data__/output/guide.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guide.xml')
-    )
-
-    const zip = new Zip()
-    const expected = zip.decompress(fs.readFileSync('tests/__data__/output/guide.xml.gz'))
-    const result = zip.decompress(
-      fs.readFileSync('tests/__data__/expected/epg_grab/guide.xml.gz')
-    )
-    expect(expected).toEqual(result)
-  }, 30000)
-
-  it('can grab epg with wildcard as output', () => {
-    const cmd = `${ENV_VAR} npm run grab --- --channels="tests/__data__/input/epg_grab/sites/example.com/example.com.channels.xml" --output="tests/__data__/output/guides/{lang}/{site}.xml"`
-    const stdout = execSync(cmd, { encoding: 'utf8' })
-    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
-
-    expect(content('tests/__data__/output/guides/en/example.com.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guides/en/example.com.xml')
-    )
-
-    expect(content('tests/__data__/output/guides/fr/example.com.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guides/fr/example.com.xml')
-    )
-  })
-
-  it('can grab epg then language filter enabled', () => {
-    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example.com/example.com.channels.xml --output=tests/__data__/output/guides/{lang}/{site}.xml --lang=fr`
-    const stdout = execSync(cmd, { encoding: 'utf8' })
-    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
-
-    expect(content('tests/__data__/output/guides/fr/example.com.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guides/fr/example.com.xml')
-    )
-  })
-
-  it('can grab epg then using a multi-language filter', () => {
-    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/example.com/example.com.channels.xml --output=tests/__data__/output/guides/{site}.xml --lang=fr,it`
-    const stdout = execSync(cmd, { encoding: 'utf8' })
-    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
-
-    expect(content('tests/__data__/output/guides/example.com.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guide_4.xml')
-    )
-  })
-
-  it('can grab epg using custom channels list', () => {
-    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/custom.channels.xml --output=tests/__data__/output/guide.xml`
-    const stdout = execSync(cmd, { encoding: 'utf8' })
-    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
-
-    expect(content('tests/__data__/output/guide.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guide_3.xml')
     )
   })
 
@@ -119,29 +47,112 @@ describe('epg:grab', () => {
     expect(errorThrown).toBe(true)
   })
 
+  it('can grab epg with wildcard as output', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels="tests/__data__/input/epg_grab/sites/example.com/example.com.channels.xml" --output="tests/__data__/output/guides/{lang}/{site}.xml" --timeout=100`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guides/en/example.com.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/guides/en/example.com.xml')
+    )
+
+    expect(content('tests/__data__/output/guides/fr/example.com.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/guides/fr/example.com.xml')
+    )
+  })
+
+  it('can grab epg then language filter enabled', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example.com/example.com.channels.xml --output=tests/__data__/output/guides/{lang}/{site}.xml --lang=fr --timeout=100`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guides/fr/example.com.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/guides/fr/example.com.xml')
+    )
+  })
+
+  it('can grab epg then using a multi-language filter', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/example.com/example.com.channels.xml --output=tests/__data__/output/guides/{site}.xml --lang=fr,it --timeout=100`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guides/example.com.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/lang.guide.xml')
+    )
+  })
+
   it('can grab epg via https proxy', () => {
     const cmd = `${ENV_VAR} npm run grab --- --site=example.com --proxy=https://bob:123456@proxy.com:1234 --output="${path.resolve(
       'tests/__data__/output/guide.xml'
-    )}"`
+    )}" --timeout=100`
     const stdout = execSync(cmd, { encoding: 'utf8' })
     if (process.env.DEBUG === 'true') console.log(cmd, stdout)
 
     expect(content('tests/__data__/output/guide.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guide_2.xml')
+      content('tests/__data__/expected/epg_grab/proxy.guide.xml')
     )
   })
 
   it('can grab epg via socks5 proxy', () => {
     const cmd = `${ENV_VAR} npm run grab --- --site=example.com --proxy=socks5://bob:123456@proxy.com:1234 --output="${path.resolve(
       'tests/__data__/output/guide.xml'
-    )}"`
+    )}" --timeout=100`
     const stdout = execSync(cmd, { encoding: 'utf8' })
     if (process.env.DEBUG === 'true') console.log(cmd, stdout)
 
     expect(content('tests/__data__/output/guide.xml')).toEqual(
-      content('tests/__data__/expected/epg_grab/guide_2.xml')
+      content('tests/__data__/expected/epg_grab/proxy.guide.xml')
     )
   })
+
+  it('can grab epg with curl option', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --site=example.com --curl --output="${path.resolve(
+      'tests/__data__/output/guide.xml'
+    )}" --timeout=100`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(stdout).toContain('curl https://example.com')
+  })
+
+  it('can grab epg with multiple channels.xml files', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/**/*.channels.xml --output=tests/__data__/output/guide.xml --timeout=100`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guide.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/template.guide.xml')
+    )
+  })
+
+  it('can grab epg using custom channels list', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/custom.channels.xml --output=tests/__data__/output/guide.xml  --timeout=100`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guide.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/custom_channels.guide.xml')
+    )
+  })
+
+  it('can grab epg with gzip option enabled', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/**/*.channels.xml --output="${path.resolve(
+      'tests/__data__/output/guide.xml'
+    )}" --gzip  --timeout=100`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guide.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/template.guide.xml')
+    )
+
+    const zip = new Zip()
+    const expected = zip.decompress(fs.readFileSync('tests/__data__/output/guide.xml.gz'))
+    const result = zip.decompress(
+      fs.readFileSync('tests/__data__/expected/epg_grab/template.guide.xml.gz')
+    )
+    expect(expected).toEqual(result)
+  })
 })
 
 function content(filepath: string) {