merge patch 1/7 to the rest, patch validate to take multiple files

This commit is contained in:
theofficialomega
2025-07-26 18:47:45 +02:00
68 changed files with 1004 additions and 1641 deletions
+4 -3
View File
@@ -18,22 +18,23 @@ jobs:
- name: changed files
id: files
run: |
git fetch origin master:master
JS_ANY_CHANGED=false
JS_ALL_CHANGED_FILES=$(git diff --name-only tests/**/*.js tests/**/*.ts scripts/**/*.js scripts/**/*.mts scripts/**/*.ts sites/**/*.js sites/**/*.ts | tr '\n' ' ')
JS_ALL_CHANGED_FILES=$(git diff --name-only master -- tests/**/*.js tests/**/*.ts scripts/**/*.js scripts/**/*.mts scripts/**/*.ts sites/**/*.js sites/**/*.ts | tr '\n' ' ')
if [ -n "${JS_ALL_CHANGED_FILES}" ]; then
JS_ANY_CHANGED=true
fi
echo "js_all_changed_files=$JS_ALL_CHANGED_FILES" >> "$GITHUB_OUTPUT"
echo "js_any_changed=$JS_ANY_CHANGED" >> "$GITHUB_OUTPUT"
CHANNELS_ANY_CHANGED=false
CHANNELS_ALL_CHANGED_FILES=$(git diff --name-only sites/**/*.channels.xml | tr '\n' ' ')
CHANNELS_ALL_CHANGED_FILES=$(git diff --name-only master -- sites/**/*.channels.xml | tr '\n' ' ')
if [ -n "${CHANNELS_ALL_CHANGED_FILES}" ]; then
CHANNELS_ANY_CHANGED=true
fi
echo "channels_all_changed_files=$CHANNELS_ALL_CHANGED_FILES" >> "$GITHUB_OUTPUT"
echo "channels_any_changed=$CHANNELS_ANY_CHANGED" >> "$GITHUB_OUTPUT"
- uses: actions/setup-node@v4
if: ${{ !env.ACT && (steps.files.outputs.js_any_changed == 'true' || steps.files.outputs.channels_any_changed == 'true') }}
if: ${{ steps.files.outputs.js_any_changed == 'true' || steps.files.outputs.channels_any_changed == 'true' }}
with:
node-version: 22
cache: 'npm'
+3 -4
View File
@@ -7,14 +7,14 @@ jobs:
main:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: tibdex/github-app-token@v1.8.2
if: ${{ !env.ACT }}
id: create-app-token
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
- uses: actions/checkout@v3
- uses: actions/checkout@v4
if: ${{ !env.ACT }}
with:
token: ${{ steps.create-app-token.outputs.token }}
@@ -22,8 +22,7 @@ jobs:
run: |
git config user.name "iptv-bot[bot]"
git config user.email "84861620+iptv-bot[bot]@users.noreply.github.com"
- uses: actions/setup-node@v3
if: ${{ !env.ACT }}
- uses: actions/setup-node@v4
with:
node-version: 22
cache: 'npm'
+8 -9
View File
@@ -65,6 +65,7 @@ Options:
--days <days> Number of days for which the program will be loaded (defaults to the value from the site config)
--maxConnections <number> Number of concurrent requests (default: 1)
--gzip Specifies whether or not to create a compressed version of the guide (default: false)
--curl Display each request as CURL (default: false)
```
### Parallel downloading
@@ -97,19 +98,15 @@ npm run grab --- --channels=path/to/custom.channels.xml
### Run on schedule
To download the guide on a schedule, you can use the included process manager. Just run it with desire [cron expression](https://crontab.guru/) and the `grab` options:
If you want to download guides on a schedule, you can use [cron](https://en.wikipedia.org/wiki/Cron) or any other task scheduler. Currently, we use a tool called `chronos` for this purpose.
To start it, you only need to specify the necessary `grab` command and [cron expression](https://crontab.guru/):
```sh
npx pm2 start npm --no-autorestart --cron-restart="0 0,12 * * *" -- run grab --- --site=example.com
npx chronos --execute="npm run grab --- --site=example.com" --pattern="0 0,12 * * *" --log
```
To track the process, you can use the command:
```sh
npx pm2 logs
```
For more info go to [pm2](https://pm2.keymetrics.io/docs/usage/quick-start/) documentation.
For more info go to [chronos](https://github.com/freearhey/chronos) documentation.
### Access the guide by URL
@@ -186,6 +183,7 @@ docker run \
-e CRON_SCHEDULE="0 0,12 * * *" \
-e MAX_CONNECTIONS=10 \
-e GZIP=true \
-e CURL=true \
-e PROXY="socks5://127.0.0.1:1234" \
-e DAYS=14 \
-e TIMEOUT=5 \
@@ -198,6 +196,7 @@ iptv-org/epg
| CRON_SCHEDULE | A [cron expression](https://crontab.guru/) describing the schedule of the guide loadings (default: "0 0 \* \* \*") |
| MAX_CONNECTIONS | Limit on the number of concurrent requests (default: 1) |
| GZIP | Boolean value indicating whether to create a compressed version of the guide (default: false) |
| CURL | Display each request as CURL (default: false) |
| PROXY | Use the specified proxy |
| DAYS | Number of days for which the guide will be loaded (defaults to the value from the site config) |
| TIMEOUT | Timeout for each request in milliseconds (default: 0) |
+6 -6
View File
@@ -73,7 +73,7 @@
<tr><td><a href="sites/guidetnt.com">guidetnt.com</a></td><td align="right">69</td><td align="right">69</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/horizon.tv">horizon.tv</a></td><td align="right">184</td><td align="right">172</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/hoy.tv">hoy.tv</a></td><td align="right">3</td><td align="right">1</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/i.mjh.nz">i.mjh.nz</a></td><td align="right">6458</td><td align="right">1488</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/i.mjh.nz">i.mjh.nz</a></td><td align="right">6458</td><td align="right">1489</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/i24news.tv">i24news.tv</a></td><td align="right">4</td><td align="right">3</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/iltalehti.fi">iltalehti.fi</a></td><td align="right">142</td><td align="right">44</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/indihometv.com">indihometv.com</a></td><td align="right">130</td><td align="right">124</td><td align="center">🟢</td><td></td></tr>
@@ -151,7 +151,7 @@
<tr><td><a href="sites/singtel.com">singtel.com</a></td><td align="right">155</td><td align="right">113</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/sjonvarp.is">sjonvarp.is</a></td><td align="right">13</td><td align="right">13</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/sky.co.nz">sky.co.nz</a></td><td align="right">111</td><td align="right">93</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/sky.com">sky.com</a></td><td align="right">559</td><td align="right">458</td><td align="center">🟢</td><td>https://github.com/iptv-org/epg/issues/2763</td></tr>
<tr><td><a href="sites/sky.com">sky.com</a></td><td align="right">559</td><td align="right">458</td><td align="center">🟡</td><td>https://github.com/iptv-org/epg/issues/2763</td></tr>
<tr><td><a href="sites/sky.de">sky.de</a></td><td align="right">75</td><td align="right">75</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/skylife.co.kr">skylife.co.kr</a></td><td align="right">251</td><td align="right">0</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/skyperfectv.co.jp">skyperfectv.co.jp</a></td><td align="right">137</td><td align="right">130</td><td align="center">🟢</td><td></td></tr>
@@ -180,7 +180,7 @@
<tr><td><a href="sites/tv-spored.siol.net">tv-spored.siol.net</a></td><td align="right">312</td><td align="right">0</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv.blue.ch">tv.blue.ch</a></td><td align="right">1030</td><td align="right">565</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv.cctv.com">tv.cctv.com</a></td><td align="right">94</td><td align="right">88</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv.dir.bg">tv.dir.bg</a></td><td align="right">111</td><td align="right">93</td><td align="center">🟢</td><td>https://github.com/iptv-org/epg/issues/2779</td></tr>
<tr><td><a href="sites/tv.dir.bg">tv.dir.bg</a></td><td align="right">111</td><td align="right">93</td><td align="center">🔴</td><td>https://github.com/iptv-org/epg/issues/2779</td></tr>
<tr><td><a href="sites/tv.lv">tv.lv</a></td><td align="right">137</td><td align="right">49</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv.magenta.at">tv.magenta.at</a></td><td align="right">307</td><td align="right">228</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv.mail.ru">tv.mail.ru</a></td><td align="right">664</td><td align="right">643</td><td align="center">🟢</td><td></td></tr>
@@ -189,7 +189,7 @@
<tr><td><a href="sites/tv.post.lu">tv.post.lu</a></td><td align="right">332</td><td align="right">242</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv.sfr.fr">tv.sfr.fr</a></td><td align="right">489</td><td align="right">456</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv.trueid.net">tv.trueid.net</a></td><td align="right">266</td><td align="right">74</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv.yandex.ru">tv.yandex.ru</a></td><td align="right">97</td><td align="right">67</td><td align="center">🟢</td><td>https://github.com/iptv-org/epg/issues/2803</td></tr>
<tr><td><a href="sites/tv.yandex.ru">tv.yandex.ru</a></td><td align="right">97</td><td align="right">67</td><td align="center">🔴</td><td>https://github.com/iptv-org/epg/issues/2803</td></tr>
<tr><td><a href="sites/tv24.co.uk">tv24.co.uk</a></td><td align="right">1072</td><td align="right">39</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv24.se">tv24.se</a></td><td align="right">326</td><td align="right">157</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tv2go.t-2.net">tv2go.t-2.net</a></td><td align="right">335</td><td align="right">254</td><td align="center">🟢</td><td></td></tr>
@@ -211,11 +211,11 @@
<tr><td><a href="sites/tvmusor.hu">tvmusor.hu</a></td><td align="right">99</td><td align="right">67</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tvmustra.hu">tvmustra.hu</a></td><td align="right">188</td><td align="right">0</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tvpassport.com">tvpassport.com</a></td><td align="right">19287</td><td align="right">2509</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tvplus.com.tr">tvplus.com.tr</a></td><td align="right">143</td><td align="right">134</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tvplus.com.tr">tvplus.com.tr</a></td><td align="right">143</td><td align="right">134</td><td align="center">🟢</td><td>https://github.com/iptv-org/epg/issues/2816</td></tr>
<tr><td><a href="sites/tvprofil.com">tvprofil.com</a></td><td align="right">5836</td><td align="right">455</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/tvtv.us">tvtv.us</a></td><td align="right">2299</td><td align="right">2255</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/v3.myafn.dodmedia.osd.mil">v3.myafn.dodmedia.osd.mil</a></td><td align="right">8</td><td align="right">8</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/vidio.com">vidio.com</a></td><td align="right">47</td><td align="right">46</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/vidio.com">vidio.com</a></td><td align="right">57</td><td align="right">52</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/virginmediatelevision.ie">virginmediatelevision.ie</a></td><td align="right">5</td><td align="right">5</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/virgintvgo.virginmedia.com">virgintvgo.virginmedia.com</a></td><td align="right">238</td><td align="right">195</td><td align="center">🟢</td><td></td></tr>
<tr><td><a href="sites/visionplus.id">visionplus.id</a></td><td align="right">250</td><td align="right">226</td><td align="center">🟢</td><td></td></tr>
+33 -967
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -18,7 +18,6 @@
"prepare": "husky"
},
"private": true,
"author": "Arhey",
"license": "UNLICENSED",
"jest": {
"setupFiles": [
@@ -40,6 +39,7 @@
"@alex_neo/jest-expect-message": "^1.0.5",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.31.0",
"@freearhey/chronos": "^0.0.1",
"@freearhey/core": "^0.10.2",
"@freearhey/search-js": "^0.1.2",
"@ntlab/sfetch": "^1.2.0",
+8 -7
View File
@@ -1,3 +1,9 @@
const grab = process.env.SITE
? `npm run grab -- --site=${process.env.SITE} ${
process.env.CLANG ? `--lang=${process.env.CLANG}` : ''
} --output=public/guide.xml`
: 'npm run grab -- --channels=channels.xml --output=public/guide.xml'
module.exports = {
apps: [
{
@@ -9,15 +15,10 @@ module.exports = {
},
{
name: 'grab',
script: process.env.SITE
? `npm run grab -- --site=${process.env.SITE} ${
process.env.CLANG ? `--lang=${process.env.CLANG}` : ''
} --output=public/guide.xml`
: 'npm run grab -- --channels=channels.xml --output=public/guide.xml',
cron_restart: process.env.CRON || null,
script: `npx chronos -e "${grab}" -p "${process.env.CRON_SCHEDULE}" -l`,
instances: 1,
watch: false,
autorestart: false
autorestart: true
}
]
}
+16 -31
View File
@@ -1,17 +1,9 @@
import { Logger, Storage, Collection } from '@freearhey/core'
import { ChannelsParser } from '../../core'
import path from 'path'
import { Logger, Collection, Storage } from '@freearhey/core'
import { SITES_DIR, API_DIR } from '../../constants'
import { GuideChannel } from '../../models'
import { ChannelsParser } from '../../core'
import epgGrabber from 'epg-grabber'
type OutputItem = {
channel: string | null
feed: string | null
site: string
site_id: string
site_name: string
lang: string
}
import path from 'path'
async function main() {
const logger = new Logger()
@@ -20,31 +12,24 @@ async function main() {
logger.info('loading channels...')
const sitesStorage = new Storage(SITES_DIR)
const parser = new ChannelsParser({ storage: sitesStorage })
const parser = new ChannelsParser({
storage: sitesStorage
})
let files: string[] = []
files = await sitesStorage.list('**/*.channels.xml')
const files: string[] = await sitesStorage.list('**/*.channels.xml')
let parsedChannels = new Collection()
const channels = new Collection()
for (const filepath of files) {
parsedChannels = parsedChannels.concat(await parser.parse(filepath))
const channelList = await parser.parse(filepath)
channelList.channels.forEach((data: epgGrabber.Channel) => {
channels.add(new GuideChannel(data))
})
}
logger.info(` found ${parsedChannels.count()} channel(s)`)
logger.info(`found ${channels.count()} channel(s)`)
const output = parsedChannels.map((channel: epgGrabber.Channel): OutputItem => {
const xmltv_id = channel.xmltv_id || ''
const [channelId, feedId] = xmltv_id.split('@')
return {
channel: channelId || null,
feed: feedId || null,
site: channel.site || '',
site_id: channel.site_id || '',
site_name: channel.name,
lang: channel.lang || ''
}
})
const output = channels.map((channel: GuideChannel) => channel.toJSON())
const apiStorage = new Storage(API_DIR)
const outputFilename = 'guides.json'
+2 -1
View File
@@ -17,7 +17,8 @@ async function main() {
loader.download('feeds.json'),
loader.download('timezones.json'),
loader.download('guides.json'),
loader.download('streams.json')
loader.download('streams.json'),
loader.download('logos.json')
])
}
+22 -23
View File
@@ -1,17 +1,17 @@
import { Storage, Collection, Logger, Dictionary } from '@freearhey/core'
import type { DataProcessorData } from '../../types/dataProcessor'
import type { DataLoaderData } from '../../types/dataLoader'
import { ChannelSearchableData } from '../../types/channel'
import { Channel, ChannelList, Feed } from '../../models'
import { DataProcessor, DataLoader } from '../../core'
import { select, input } from '@inquirer/prompts'
import { ChannelsParser, XML } from '../../core'
import { Channel, Feed } from '../../models'
import { ChannelsParser } from '../../core'
import { DATA_DIR } from '../../constants'
import nodeCleanup from 'node-cleanup'
import sjs from '@freearhey/search-js'
import epgGrabber from 'epg-grabber'
import { Command } from 'commander'
import readline from 'readline'
import sjs from '@freearhey/search-js'
import { DataProcessor, DataLoader } from '../../core'
import type { DataLoaderData } from '../../types/dataLoader'
import type { DataProcessorData } from '../../types/dataProcessor'
import epgGrabber from 'epg-grabber'
import { ChannelSearchableData } from '../../types/channel'
type ChoiceValue = { type: string; value?: Feed | Channel }
type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean }
@@ -34,11 +34,11 @@ program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(proc
const filepath = program.args[0]
const logger = new Logger()
const storage = new Storage()
let parsedChannels = new Collection()
let channelList = new ChannelList({ channels: [] })
main(filepath)
nodeCleanup(() => {
save(filepath)
save(filepath, channelList)
})
export default async function main(filepath: string) {
@@ -51,18 +51,18 @@ export default async function main(filepath: string) {
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const { feedsGroupedByChannelId, channels, channelsKeyById }: DataProcessorData =
const { channels, channelsKeyById, feedsGroupedByChannelId }: DataProcessorData =
processor.process(data)
logger.info('loading channels...')
const parser = new ChannelsParser({ storage })
parsedChannels = await parser.parse(filepath)
const parsedChannelsWithoutId = parsedChannels.filter(
channelList = await parser.parse(filepath)
const parsedChannelsWithoutId = channelList.channels.filter(
(channel: epgGrabber.Channel) => !channel.xmltv_id
)
logger.info(
`found ${parsedChannels.count()} channels (including ${parsedChannelsWithoutId.count()} without ID)`
`found ${channelList.channels.count()} channels (including ${parsedChannelsWithoutId.count()} without ID)`
)
logger.info('creating search index...')
@@ -73,10 +73,10 @@ export default async function main(filepath: string) {
logger.info('starting...\n')
for (const parsedChannel of parsedChannelsWithoutId.all()) {
for (const channel of parsedChannelsWithoutId.all()) {
try {
parsedChannel.xmltv_id = await selectChannel(
parsedChannel,
channel.xmltv_id = await selectChannel(
channel,
searchIndex,
feedsGroupedByChannelId,
channelsKeyById
@@ -124,8 +124,8 @@ async function selectChannel(
case 'channel': {
const selectedChannel = selected.value
if (!selectedChannel) return ''
const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId)
if (selectedFeedId === '-') return selectedChannel.id
const selectedFeedId = await selectFeed(selectedChannel.id || '', feedsGroupedByChannelId)
if (selectedFeedId === '-') return selectedChannel.id || ''
return [selectedChannel.id, selectedFeedId].join('@')
}
}
@@ -153,7 +153,7 @@ async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary
case 'feed':
const selectedFeed = selected.value
if (!selectedFeed) return ''
return selectedFeed.id
return selectedFeed.id || ''
}
return ''
@@ -205,10 +205,9 @@ function getFeedChoises(feeds: Collection): Choice[] {
return choises
}
function save(filepath: string) {
function save(filepath: string, channelList: ChannelList) {
if (!storage.existsSync(filepath)) return
const xml = new XML(parsedChannels)
storage.saveSync(filepath, xml.toString())
storage.saveSync(filepath, channelList.toString())
logger.info(`\nFile '${filepath}' successfully saved`)
}
+1
View File
@@ -19,6 +19,7 @@ const xsd = `<?xml version="1.0" encoding="UTF-8"?>
<xs:attribute use="required" ref="site_id"/>
<xs:attribute name="xmltv_id" use="required" type="xs:string"/>
<xs:attribute name="logo" type="xs:string"/>
<xs:attribute name="lcn" type="xs:string"/>
</xs:complexType>
</xs:element>
<xs:attribute name="site">
+25 -30
View File
@@ -1,8 +1,9 @@
import { Logger, File, Collection, Storage } from '@freearhey/core'
import { ChannelsParser, XML } from '../../core'
import { Channel } from 'epg-grabber'
import { Command } from 'commander'
import { Logger, File, Storage } from '@freearhey/core'
import { ChannelsParser } from '../../core'
import { ChannelList } from '../../models'
import { pathToFileURL } from 'node:url'
import epgGrabber from 'epg-grabber'
import { Command } from 'commander'
const program = new Command()
program
@@ -21,17 +22,25 @@ type ParseOptions = {
const options: ParseOptions = program.opts()
async function main() {
function isPromise(promise: object[] | Promise<object[]>) {
return (
!!promise &&
typeof promise === 'object' &&
typeof (promise as Promise<object[]>).then === 'function'
)
}
const storage = new Storage()
const parser = new ChannelsParser({ storage })
const logger = new Logger()
const parser = new ChannelsParser({ storage })
const file = new File(options.config)
const dir = file.dirname()
const config = (await import(pathToFileURL(options.config).toString())).default
const outputFilepath = options.output || `${dir}/${config.site}.channels.xml`
let channels = new Collection()
let channelList = new ChannelList({ channels: [] })
if (await storage.exists(outputFilepath)) {
channels = await parser.parse(outputFilepath)
channelList = await parser.parse(outputFilepath)
}
const args: {
@@ -49,45 +58,31 @@ async function main() {
if (isPromise(parsedChannels)) {
parsedChannels = await parsedChannels
}
parsedChannels = parsedChannels.map((channel: Channel) => {
parsedChannels = parsedChannels.map((channel: epgGrabber.Channel) => {
channel.site = config.site
return channel
})
let output = new Collection()
parsedChannels.forEach((channel: Channel) => {
const found: Channel | undefined = channels.first(
(_channel: Channel) => _channel.site_id == channel.site_id
)
const newChannelList = new ChannelList({ channels: [] })
parsedChannels.forEach((channel: epgGrabber.Channel) => {
if (!channel.site_id) return
const found: epgGrabber.Channel | undefined = channelList.get(channel.site_id)
if (found) {
channel.xmltv_id = found.xmltv_id
channel.lang = found.lang
}
output.add(channel)
newChannelList.add(channel)
})
output = output.orderBy([
(channel: Channel) => channel.lang || '_',
(channel: Channel) => (channel.xmltv_id ? channel.xmltv_id.toLowerCase() : '0'),
(channel: Channel) => channel.site_id
])
newChannelList.sort()
const xml = new XML(output)
await storage.save(outputFilepath, xml.toString())
await storage.save(outputFilepath, newChannelList.toString())
logger.info(`File '${outputFilepath}' successfully saved`)
}
main()
function isPromise(promise: object[] | Promise<object[]>) {
return (
!!promise &&
typeof promise === 'object' &&
typeof (promise as Promise<object[]>).then === 'function'
)
}
+29 -16
View File
@@ -1,16 +1,18 @@
import { Storage, Collection, Dictionary, File } from '@freearhey/core'
import { ChannelsParser } from '../../core'
import { Channel } from '../../models'
import { ChannelsParser, DataLoader, DataProcessor } from '../../core'
import { DataProcessorData } from '../../types/dataProcessor'
import { Storage, Dictionary, File } from '@freearhey/core'
import { DataLoaderData } from '../../types/dataLoader'
import { ChannelList } from '../../models'
import { DATA_DIR } from '../../constants'
import epgGrabber from 'epg-grabber'
import { program } from 'commander'
import chalk from 'chalk'
import langs from 'langs'
import { DATA_DIR } from '../../constants'
import epgGrabber from 'epg-grabber'
program.argument('[filepath]', 'Path to *.channels.xml files to validate').parse(process.argv)
type ValidationError = {
type: 'duplicate' | 'wrong_xmltv_id' | 'wrong_lang'
type: 'duplicate' | 'wrong_channel_id' | 'wrong_feed_id' | 'wrong_lang'
name: string
lang?: string
xmltv_id?: string
@@ -19,12 +21,14 @@ type ValidationError = {
}
async function main() {
const parser = new ChannelsParser({ storage: new Storage() })
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const channelsData = await dataStorage.json('channels.json')
const channels = new Collection(channelsData).map(data => new Channel(data))
const channelsGroupedById = channels.groupBy((channel: Channel) => channel.id)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const { channelsKeyById, feedsKeyByStreamId }: DataProcessorData = processor.process(data)
const parser = new ChannelsParser({
storage: new Storage()
})
let totalFiles = 0
let totalErrors = 0
@@ -35,11 +39,11 @@ async function main() {
const file = new File(filepath)
if (file.extension() !== 'xml') continue
const parsedChannels = await parser.parse(filepath)
const channelList: ChannelList = await parser.parse(filepath)
const bufferBySiteId = new Dictionary()
const errors: ValidationError[] = []
parsedChannels.forEach((channel: epgGrabber.Channel) => {
channelList.channels.forEach((channel: epgGrabber.Channel) => {
const bufferId: string = channel.site_id
if (bufferBySiteId.missing(bufferId)) {
bufferBySiteId.set(bufferId, true)
@@ -54,12 +58,21 @@ async function main() {
}
if (!channel.xmltv_id) return
const [channelId] = channel.xmltv_id.split('@')
const foundChannel = channelsGroupedById.get(channelId)
const [channelId, feedId] = channel.xmltv_id.split('@')
const foundChannel = channelsKeyById.get(channelId)
if (!foundChannel) {
errors.push({ type: 'wrong_xmltv_id', ...channel })
errors.push({ type: 'wrong_channel_id', ...channel })
totalErrors++
}
if (feedId) {
const foundFeed = feedsKeyByStreamId.get(channel.xmltv_id)
if (!foundFeed) {
errors.push({ type: 'wrong_feed_id', ...channel })
totalErrors++
}
}
})
if (errors.length) {
+16 -9
View File
@@ -1,9 +1,10 @@
import { Logger, Timer, Storage, Collection } from '@freearhey/core'
import { Option, program } from 'commander'
import { QueueCreator, Job, ChannelsParser } from '../../core'
import { Option, program } from 'commander'
import { SITES_DIR } from '../../constants'
import { Channel } from 'epg-grabber'
import path from 'path'
import { SITES_DIR } from '../../constants'
import { ChannelList } from '../../models'
program
.addOption(new Option('-s, --site <name>', 'Name of the site to parse'))
@@ -44,6 +45,7 @@ program
.default(false)
.env('GZIP')
)
.addOption(new Option('--curl', 'Display each request as CURL').default(false).env('CURL'))
.parse()
export type GrabOptions = {
@@ -51,6 +53,7 @@ export type GrabOptions = {
channels?: string
output: string
gzip: boolean
curl: boolean
maxConnections: number
timeout?: string
delay?: string
@@ -85,31 +88,35 @@ async function main() {
files = await storage.list(options.channels)
}
let parsedChannels = new Collection()
let channels = new Collection()
for (const filepath of files) {
parsedChannels = parsedChannels.concat(await parser.parse(filepath))
const channelList: ChannelList = await parser.parse(filepath)
channels = channels.concat(channelList.channels)
}
if (options.lang) {
parsedChannels = parsedChannels.filter((channel: Channel) => {
channels = channels.filter((channel: Channel) => {
if (!options.lang || !channel.lang) return true
return options.lang.includes(channel.lang)
})
}
logger.info(` found ${parsedChannels.count()} channel(s)`)
logger.info(` found ${channels.count()} channel(s)`)
logger.info('run:')
runJob({ logger, parsedChannels })
runJob({ logger, channels })
}
main()
async function runJob({ logger, parsedChannels }: { logger: Logger; parsedChannels: Collection }) {
async function runJob({ logger, channels }: { logger: Logger; channels: Collection }) {
const timer = new Timer()
timer.start()
const queueCreator = new QueueCreator({
parsedChannels,
channels,
logger,
options
})
+17 -11
View File
@@ -1,21 +1,25 @@
import { IssueLoader, HTMLTable, ChannelsParser } from '../../core'
import { Logger, Storage, Collection } from '@freearhey/core'
import { ChannelList, Issue, Site } from '../../models'
import { SITES_DIR, ROOT_DIR } from '../../constants'
import { Issue, Site } from '../../models'
import { Channel } from 'epg-grabber'
async function main() {
const logger = new Logger({ disabled: true })
const loader = new IssueLoader()
const logger = new Logger({ level: -999 })
const issueLoader = new IssueLoader()
const sitesStorage = new Storage(SITES_DIR)
const channelsParser = new ChannelsParser({ storage: sitesStorage })
const sites = new Collection()
logger.info('loading channels...')
const channelsParser = new ChannelsParser({
storage: sitesStorage
})
logger.info('loading list of sites')
const folders = await sitesStorage.list('*/')
logger.info('loading issues...')
const issues = await loader.load()
const issues = await issueLoader.load()
logger.info('putting the data together...')
const brokenGuideReports = issues.filter(issue =>
@@ -33,19 +37,21 @@ async function main() {
const files = await sitesStorage.list(`${domain}/*.channels.xml`)
for (const filepath of files) {
const channels = await channelsParser.parse(filepath)
const channelList: ChannelList = await channelsParser.parse(filepath)
site.totalChannels += channels.count()
site.markedChannels += channels.filter((channel: Channel) => channel.xmltv_id).count()
site.totalChannels += channelList.channels.count()
site.markedChannels += channelList.channels
.filter((channel: Channel) => channel.xmltv_id)
.count()
}
sites.add(site)
}
logger.info('creating sites table...')
const data = new Collection()
const tableData = new Collection()
sites.forEach((site: Site) => {
data.add([
tableData.add([
{ value: `<a href="sites/${site.domain}">${site.domain}</a>` },
{ value: site.totalChannels, align: 'right' },
{ value: site.markedChannels, align: 'right' },
@@ -55,7 +61,7 @@ async function main() {
})
logger.info('updating sites.md...')
const table = new HTMLTable(data.all(), [
const table = new HTMLTable(tableData.all(), [
{ name: 'Site', align: 'left' },
{ name: 'Channels<br>(total / with xmltv-id)', colspan: 2, align: 'left' },
{ name: 'Status', align: 'left' },
+5 -7
View File
@@ -1,5 +1,6 @@
import { parseChannels } from 'epg-grabber'
import { Storage, Collection } from '@freearhey/core'
import { Storage } from '@freearhey/core'
import { ChannelList } from '../models'
type ChannelsParserProps = {
storage: Storage
@@ -12,13 +13,10 @@ export class ChannelsParser {
this.storage = storage
}
async parse(filepath: string) {
let parsedChannels = new Collection()
async parse(filepath: string): Promise<ChannelList> {
const content = await this.storage.load(filepath)
const channels = parseChannels(content)
parsedChannels = parsedChannels.concat(new Collection(channels))
const parsed = parseChannels(content)
return parsedChannels
return new ChannelList({ channels: parsed })
}
}
+6 -3
View File
@@ -49,7 +49,8 @@ export class DataLoader {
feeds,
timezones,
guides,
streams
streams,
logos
] = await Promise.all([
this.storage.json('countries.json'),
this.storage.json('regions.json'),
@@ -61,7 +62,8 @@ export class DataLoader {
this.storage.json('feeds.json'),
this.storage.json('timezones.json'),
this.storage.json('guides.json'),
this.storage.json('streams.json')
this.storage.json('streams.json'),
this.storage.json('logos.json')
])
return {
@@ -75,7 +77,8 @@ export class DataLoader {
feeds,
timezones,
guides,
streams
streams,
logos
}
}
+26 -9
View File
@@ -1,6 +1,6 @@
import { Channel, Feed, GuideChannel, Logo, Stream } from '../models'
import { DataLoaderData } from '../types/dataLoader'
import { Collection } from '@freearhey/core'
import { Channel, Feed, Guide, Stream } from '../models'
export class DataProcessor {
constructor() {}
@@ -9,31 +9,48 @@ export class DataProcessor {
let channels = new Collection(data.channels).map(data => new Channel(data))
const channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
const guides = new Collection(data.guides).map(data => new Guide(data))
const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId())
const guideChannels = new Collection(data.guides).map(data => new GuideChannel(data))
const guideChannelsGroupedByStreamId = guideChannels.groupBy((channel: GuideChannel) =>
channel.getStreamId()
)
const streams = new Collection(data.streams).map(data => new Stream(data))
const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
const feeds = new Collection(data.feeds).map(data =>
let feeds = new Collection(data.feeds).map(data =>
new Feed(data)
.withGuides(guidesGroupedByStreamId)
.withGuideChannels(guideChannelsGroupedByStreamId)
.withStreams(streamsGroupedById)
.withChannel(channelsKeyById)
)
const feedsKeyByStreamId = feeds.keyBy((feed: Feed) => feed.getStreamId())
const logos = new Collection(data.logos).map(data =>
new Logo(data).withFeed(feedsKeyByStreamId)
)
const logosGroupedByChannelId = logos.groupBy((logo: Logo) => logo.channelId)
const logosGroupedByStreamId = logos.groupBy((logo: Logo) => logo.getStreamId())
feeds = feeds.map((feed: Feed) => feed.withLogos(logosGroupedByStreamId))
const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
channels = channels.map((channel: Channel) => channel.withFeeds(feedsGroupedByChannelId))
channels = channels.map((channel: Channel) =>
channel.withFeeds(feedsGroupedByChannelId).withLogos(logosGroupedByChannelId)
)
return {
guideChannelsGroupedByStreamId,
feedsGroupedByChannelId,
guidesGroupedByStreamId,
logosGroupedByChannelId,
logosGroupedByStreamId,
streamsGroupedById,
feedsKeyByStreamId,
channelsKeyById,
guideChannels,
channels,
streams,
guides,
feeds
feeds,
logos
}
}
}
+4
View File
@@ -70,6 +70,10 @@ export class Grabber {
}
}
if (this.options.curl === true) {
config.curl = true
}
const _programs = await this.grabber.grab(
channel,
date,
-60
View File
@@ -1,60 +0,0 @@
import { Collection, Logger, DateTime, Storage, Zip } from '@freearhey/core'
import { Channel } from 'epg-grabber'
import { XMLTV } from '../core'
import path from 'path'
type GuideProps = {
channels: Collection
programs: Collection
logger: Logger
filepath: string
gzip: boolean
}
export class Guide {
channels: Collection
programs: Collection
logger: Logger
storage: Storage
filepath: string
gzip: boolean
constructor({ channels, programs, logger, filepath, gzip }: GuideProps) {
this.channels = channels
this.programs = programs
this.logger = logger
this.storage = new Storage(path.dirname(filepath))
this.filepath = filepath
this.gzip = gzip || false
}
async save() {
const channels = this.channels.uniqBy(
(channel: Channel) => `${channel.xmltv_id}:${channel.site}`
)
const programs = this.programs
const currDate = new DateTime(process.env.CURR_DATE || new Date().toISOString(), {
timezone: 'UTC'
})
const xmltv = new XMLTV({
channels,
programs,
date: currDate
})
const xmlFilepath = this.filepath
const xmlFilename = path.basename(xmlFilepath)
this.logger.info(` saving to "${xmlFilepath}"...`)
await this.storage.save(xmlFilename, xmltv.toString())
if (this.gzip) {
const zip = new Zip()
const compressed = zip.compress(xmltv.toString())
const gzFilepath = `${this.filepath}.gz`
const gzFilename = path.basename(gzFilepath)
this.logger.info(` saving to "${gzFilepath}"...`)
await this.storage.save(gzFilename, compressed)
}
}
}
+63 -13
View File
@@ -1,7 +1,12 @@
import { Collection, Logger, Storage, StringTemplate } from '@freearhey/core'
import { Collection, Logger, Zip, Storage, StringTemplate } from '@freearhey/core'
import epgGrabber from 'epg-grabber'
import { OptionValues } from 'commander'
import { Channel, Program } from 'epg-grabber'
import { Guide } from '.'
import { Channel, Feed, Guide } from '../models'
import path from 'path'
import { DataLoader, DataProcessor } from '.'
import { DataLoaderData } from '../types/dataLoader'
import { DataProcessorData } from '../types/dataProcessor'
import { DATA_DIR } from '../constants'
type GuideManagerProps = {
options: OptionValues
@@ -12,7 +17,6 @@ type GuideManagerProps = {
export class GuideManager {
options: OptionValues
storage: Storage
logger: Logger
channels: Collection
programs: Collection
@@ -22,22 +26,51 @@ export class GuideManager {
this.logger = logger
this.channels = channels
this.programs = programs
this.storage = new Storage()
}
async createGuides() {
const pathTemplate = new StringTemplate(this.options.output)
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const { feedsKeyByStreamId, channelsKeyById }: DataProcessorData = processor.process(data)
const groupedChannels = this.channels
.orderBy([(channel: Channel) => channel.xmltv_id])
.uniqBy((channel: Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`)
.groupBy((channel: Channel) => {
.map((channel: epgGrabber.Channel) => {
if (channel.xmltv_id && !channel.icon) {
const foundFeed: Feed = feedsKeyByStreamId.get(channel.xmltv_id)
if (foundFeed && foundFeed.hasLogo()) {
channel.icon = foundFeed.getLogoUrl()
} else {
const [channelId] = channel.xmltv_id.split('@')
const foundChannel: Channel = channelsKeyById.get(channelId)
if (foundChannel && foundChannel.hasLogo()) {
channel.icon = foundChannel.getLogoUrl()
}
}
}
return channel
})
.orderBy([
(channel: epgGrabber.Channel) => channel.index,
(channel: epgGrabber.Channel) => channel.xmltv_id
])
.uniqBy(
(channel: epgGrabber.Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`
)
.groupBy((channel: epgGrabber.Channel) => {
return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' })
})
const groupedPrograms = this.programs
.orderBy([(program: Program) => program.channel, (program: Program) => program.start])
.groupBy((program: Program) => {
.orderBy([
(program: epgGrabber.Program) => program.channel,
(program: epgGrabber.Program) => program.start
])
.groupBy((program: epgGrabber.Program) => {
const lang =
program.titles && program.titles.length && program.titles[0].lang
? program.titles[0].lang
@@ -51,11 +84,28 @@ export class GuideManager {
filepath: groupKey,
gzip: this.options.gzip,
channels: new Collection(groupedChannels.get(groupKey)),
programs: new Collection(groupedPrograms.get(groupKey)),
logger: this.logger
programs: new Collection(groupedPrograms.get(groupKey))
})
await guide.save()
await this.save(guide)
}
}
async save(guide: Guide) {
const storage = new Storage(path.dirname(guide.filepath))
const xmlFilepath = guide.filepath
const xmlFilename = path.basename(xmlFilepath)
this.logger.info(` saving to "${xmlFilepath}"...`)
const xmltv = guide.toString()
await storage.save(xmlFilename, xmltv)
if (guide.gzip) {
const zip = new Zip()
const compressed = zip.compress(xmltv)
const gzFilepath = `${guide.filepath}.gz`
const gzFilename = path.basename(gzFilepath)
this.logger.info(` saving to "${gzFilepath}"...`)
await storage.save(gzFilename, compressed)
}
}
}
-3
View File
@@ -4,7 +4,6 @@ export * from './configLoader'
export * from './dataLoader'
export * from './dataProcessor'
export * from './grabber'
export * from './guide'
export * from './guideManager'
export * from './htmlTable'
export * from './issueLoader'
@@ -13,5 +12,3 @@ export * from './job'
export * from './proxyParser'
export * from './queue'
export * from './queueCreator'
export * from './xml'
export * from './xmltv'
+1
View File
@@ -23,6 +23,7 @@ export class IssueLoader {
repo: REPO,
per_page: 100,
labels,
state: 'open',
headers: {
'X-GitHub-Api-Version': '2022-11-28'
}
+12 -8
View File
@@ -2,9 +2,9 @@ import { URL } from 'node:url'
type ProxyParserResult = {
protocol: string | null
auth: {
username: string | null
password: string | null
auth?: {
username?: string
password?: string
}
host: string
port: number | null
@@ -14,14 +14,18 @@ export class ProxyParser {
parse(_url: string): ProxyParserResult {
const parsed = new URL(_url)
return {
const result: ProxyParserResult = {
protocol: parsed.protocol.replace(':', '') || null,
auth: {
username: parsed.username || null,
password: parsed.password || null
},
host: parsed.hostname,
port: parsed.port ? parseInt(parsed.port) : null
}
if (parsed.username || parsed.password) {
result.auth = {}
if (parsed.username) result.auth.username = parsed.username
if (parsed.password) result.auth.password = parsed.password
}
return result
}
}
+10 -24
View File
@@ -1,15 +1,14 @@
import { Storage, Collection, DateTime, Logger } from '@freearhey/core'
import { ChannelsParser, ConfigLoader, Queue } from './'
import { SITES_DIR, DATA_DIR } from '../constants'
import { GrabOptions } from '../commands/epg/grab'
import { ConfigLoader, Queue } from './'
import { SiteConfig } from 'epg-grabber'
import path from 'path'
import { GrabOptions } from '../commands/epg/grab'
import { Channel } from '../models'
type QueueCreatorProps = {
logger: Logger
options: GrabOptions
parsedChannels: Collection
channels: Collection
}
export class QueueCreator {
@@ -17,42 +16,29 @@ export class QueueCreator {
logger: Logger
sitesStorage: Storage
dataStorage: Storage
parser: ChannelsParser
parsedChannels: Collection
channels: Collection
options: GrabOptions
constructor({ parsedChannels, logger, options }: QueueCreatorProps) {
this.parsedChannels = parsedChannels
constructor({ channels, logger, options }: QueueCreatorProps) {
this.channels = channels
this.logger = logger
this.sitesStorage = new Storage()
this.dataStorage = new Storage(DATA_DIR)
this.parser = new ChannelsParser({ storage: new Storage() })
this.options = options
this.configLoader = new ConfigLoader()
}
async create(): Promise<Queue> {
const channelsContent = await this.dataStorage.json('channels.json')
const channels = new Collection(channelsContent).map(data => new Channel(data))
let index = 0
const queue = new Queue()
for (const channel of this.parsedChannels.all()) {
for (const channel of this.channels.all()) {
channel.index = index++
if (!channel.site || !channel.site_id || !channel.name) continue
const configPath = path.resolve(SITES_DIR, `${channel.site}/${channel.site}.config.js`)
const config: SiteConfig = await this.configLoader.load(configPath)
if (channel.xmltv_id) {
if (!channel.icon) {
const found: Channel = channels.first(
(_channel: Channel) => _channel.id === channel.xmltv_id
)
if (found) {
channel.icon = found.logo
}
}
} else {
if (!channel.xmltv_id) {
channel.xmltv_id = channel.site_id
}
-56
View File
@@ -1,56 +0,0 @@
import { Collection } from '@freearhey/core'
import { Channel } from 'epg-grabber'
export class XML {
items: Collection
constructor(items: Collection) {
this.items = items
}
toString() {
let output = '<?xml version="1.0" encoding="UTF-8"?>\r\n<channels>\r\n'
this.items.forEach((channel: Channel) => {
const logo = channel.logo ? ` logo="${channel.logo}"` : ''
const xmltv_id = channel.xmltv_id || ''
const lang = channel.lang || ''
const site_id = channel.site_id || ''
output += ` <channel site="${channel.site}" lang="${lang}" xmltv_id="${escapeString(
xmltv_id
)}" site_id="${site_id}"${logo}>${escapeString(channel.name)}</channel>\r\n`
})
output += '</channels>\r\n'
return output
}
}
function escapeString(value: string, defaultValue: string = '') {
if (!value) return defaultValue
const regex = new RegExp(
'((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
'|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
'[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
'(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
'g'
)
value = String(value || '').replace(regex, '')
return value
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&apos;')
.replace(/\n|\r/g, ' ')
.replace(/ +/g, ' ')
.trim()
}
-28
View File
@@ -1,28 +0,0 @@
import { DateTime, Collection } from '@freearhey/core'
import { generateXMLTV } from 'epg-grabber'
type XMLTVProps = {
channels: Collection
programs: Collection
date: DateTime
}
export class XMLTV {
channels: Collection
programs: Collection
date: DateTime
constructor({ channels, programs, date }: XMLTVProps) {
this.channels = channels
this.programs = programs
this.date = date
}
toString() {
return generateXMLTV({
channels: this.channels.all(),
programs: this.programs.all(),
date: this.date.toJSON()
})
}
}
+68 -17
View File
@@ -1,26 +1,28 @@
import { ChannelData, ChannelSearchableData } from '../types/channel'
import { Collection, Dictionary } from '@freearhey/core'
import { Stream, Guide, Feed } from './'
import { Stream, Feed, Logo, GuideChannel } from './'
export class Channel {
id: string
name: string
id?: string
name?: string
altNames?: Collection
network?: string
owners?: Collection
countryCode: string
countryCode?: string
subdivisionCode?: string
cityName?: string
categoryIds?: Collection
isNSFW: boolean
isNSFW: boolean = false
launched?: string
closed?: string
replacedBy?: string
website?: string
logo?: string
feeds?: Collection
logos: Collection = new Collection()
constructor(data?: ChannelData) {
if (!data) return
constructor(data: ChannelData) {
this.id = data.id
this.name = data.name
this.altNames = new Collection(data.alt_names)
@@ -35,11 +37,16 @@ export class Channel {
this.closed = data.closed || undefined
this.replacedBy = data.replaced_by || undefined
this.website = data.website || undefined
this.logo = data.logo
}
withFeeds(feedsGroupedByChannelId: Dictionary): this {
this.feeds = new Collection(feedsGroupedByChannelId.get(this.id))
if (this.id) this.feeds = new Collection(feedsGroupedByChannelId.get(this.id))
return this
}
withLogos(logosGroupedByChannelId: Dictionary): this {
if (this.id) this.logos = new Collection(logosGroupedByChannelId.get(this.id))
return this
}
@@ -50,19 +57,19 @@ export class Channel {
return this.feeds
}
getGuides(): Collection {
let guides = new Collection()
getGuideChannels(): Collection {
let channels = new Collection()
this.getFeeds().forEach((feed: Feed) => {
guides = guides.concat(feed.getGuides())
channels = channels.concat(feed.getGuideChannels())
})
return guides
return channels
}
getGuideNames(): Collection {
return this.getGuides()
.map((guide: Guide) => guide.siteName)
getGuideChannelNames(): Collection {
return this.getGuideChannels()
.map((channel: GuideChannel) => channel.siteName)
.uniq()
}
@@ -100,12 +107,56 @@ export class Channel {
return this.altNames || new Collection()
}
getLogos(): Collection {
function feed(logo: Logo): number {
if (!logo.feed) return 1
if (logo.feed.isMain) return 1
return 0
}
function format(logo: Logo): number {
const levelByFormat: { [key: string]: number } = {
SVG: 0,
PNG: 3,
APNG: 1,
WebP: 1,
AVIF: 1,
JPEG: 2,
GIF: 1
}
return logo.format ? levelByFormat[logo.format] : 0
}
function size(logo: Logo): number {
return Math.abs(512 - logo.width) + Math.abs(512 - logo.height)
}
return this.logos.orderBy([feed, format, size], ['desc', 'desc', 'asc'], false)
}
getLogo(): Logo | undefined {
return this.getLogos().first()
}
hasLogo(): boolean {
return this.getLogos().notEmpty()
}
getLogoUrl(): string {
const logo = this.getLogo()
if (!logo) return ''
return logo.url || ''
}
getSearchable(): ChannelSearchableData {
return {
id: this.getId(),
name: this.getName(),
altNames: this.getAltNames().all(),
guideNames: this.getGuideNames().all(),
guideNames: this.getGuideChannelNames().all(),
streamNames: this.getStreamNames().all(),
feedFullNames: this.getFeedFullNames().all()
}
+77
View File
@@ -0,0 +1,77 @@
import { Collection } from '@freearhey/core'
import epgGrabber from 'epg-grabber'
export class ChannelList {
channels: Collection = new Collection()
constructor(data: { channels: epgGrabber.Channel[] }) {
this.channels = new Collection(data.channels)
}
add(channel: epgGrabber.Channel): this {
this.channels.add(channel)
return this
}
get(siteId: string): epgGrabber.Channel | undefined {
return this.channels.find((channel: epgGrabber.Channel) => channel.site_id == siteId)
}
sort(): this {
this.channels = this.channels.orderBy([
(channel: epgGrabber.Channel) => channel.lang || '_',
(channel: epgGrabber.Channel) => (channel.xmltv_id ? channel.xmltv_id.toLowerCase() : '0'),
(channel: epgGrabber.Channel) => channel.site_id
])
return this
}
toString() {
function escapeString(value: string, defaultValue: string = '') {
if (!value) return defaultValue
const regex = new RegExp(
'((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
'|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
'[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
'(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
'g'
)
value = String(value || '').replace(regex, '')
return value
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&apos;')
.replace(/\n|\r/g, ' ')
.replace(/ +/g, ' ')
.trim()
}
let output = '<?xml version="1.0" encoding="UTF-8"?>\r\n<channels>\r\n'
this.channels.forEach((channel: epgGrabber.Channel) => {
const logo = channel.logo ? ` logo="${channel.logo}"` : ''
const xmltv_id = channel.xmltv_id ? escapeString(channel.xmltv_id) : ''
const lang = channel.lang || ''
const site_id = channel.site_id || ''
const site = channel.site || ''
const displayName = channel.name ? escapeString(channel.name) : ''
output += ` <channel site="${site}" lang="${lang}" xmltv_id="${xmltv_id}" site_id="${site_id}"${logo}>${displayName}</channel>\r\n`
})
output += '</channels>\r\n'
return output
}
}
+60 -8
View File
@@ -1,6 +1,6 @@
import { Collection, Dictionary } from '@freearhey/core'
import { FeedData } from '../types/feed'
import { Channel } from './channel'
import { Logo, Channel } from '.'
export class Feed {
channelId: string
@@ -12,8 +12,9 @@ export class Feed {
languageCodes: Collection
timezoneIds: Collection
videoFormat: string
guides?: Collection
guideChannels?: Collection
streams?: Collection
logos: Collection = new Collection()
constructor(data: FeedData) {
this.channelId = data.channel
@@ -42,20 +43,30 @@ export class Feed {
return this
}
withGuides(guidesGroupedByStreamId: Dictionary): this {
this.guides = new Collection(guidesGroupedByStreamId.get(`${this.channelId}@${this.id}`))
withGuideChannels(guideChannelsGroupedByStreamId: Dictionary): this {
this.guideChannels = new Collection(
guideChannelsGroupedByStreamId.get(`${this.channelId}@${this.id}`)
)
if (this.isMain) {
this.guides = this.guides.concat(new Collection(guidesGroupedByStreamId.get(this.channelId)))
this.guideChannels = this.guideChannels.concat(
new Collection(guideChannelsGroupedByStreamId.get(this.channelId))
)
}
return this
}
getGuides(): Collection {
if (!this.guides) return new Collection()
withLogos(logosGroupedByStreamId: Dictionary): this {
this.logos = new Collection(logosGroupedByStreamId.get(this.getStreamId()))
return this.guides
return this
}
getGuideChannels(): Collection {
if (!this.guideChannels) return new Collection()
return this.guideChannels
}
getStreams(): Collection {
@@ -69,4 +80,45 @@ export class Feed {
return `${this.channel.name} ${this.name}`
}
getStreamId(): string {
return `${this.channelId}@${this.id}`
}
getLogos(): Collection {
function format(logo: Logo): number {
const levelByFormat: { [key: string]: number } = {
SVG: 0,
PNG: 3,
APNG: 1,
WebP: 1,
AVIF: 1,
JPEG: 2,
GIF: 1
}
return logo.format ? levelByFormat[logo.format] : 0
}
function size(logo: Logo): number {
return Math.abs(512 - logo.width) + Math.abs(512 - logo.height)
}
return this.logos.orderBy([format, size], ['desc', 'asc'], false)
}
getLogo(): Logo | undefined {
return this.getLogos().first()
}
hasLogo(): boolean {
return this.getLogos().notEmpty()
}
getLogoUrl(): string {
const logo = this.getLogo()
if (!logo) return ''
return logo.url || ''
}
}
+27 -27
View File
@@ -1,35 +1,35 @@
import type { GuideData } from '../types/guide'
import { v4 as uuidv4 } from 'uuid'
import { Collection, DateTime } from '@freearhey/core'
import { generateXMLTV } from 'epg-grabber'
type GuideData = {
channels: Collection
programs: Collection
filepath: string
gzip: boolean
}
export class Guide {
channelId?: string
feedId?: string
siteDomain?: string
siteId?: string
siteName?: string
languageCode?: string
channels: Collection
programs: Collection
filepath: string
gzip: boolean
constructor(data?: GuideData) {
if (!data) return
this.channelId = data.channel
this.feedId = data.feed
this.siteDomain = data.site
this.siteId = data.site_id
this.siteName = data.site_name
this.languageCode = data.lang
constructor({ channels, programs, filepath, gzip }: GuideData) {
this.channels = channels
this.programs = programs
this.filepath = filepath
this.gzip = gzip || false
}
getUUID(): string {
if (!this.getStreamId() || !this.siteId) return uuidv4()
toString() {
const currDate = new DateTime(process.env.CURR_DATE || new Date().toISOString(), {
timezone: 'UTC'
})
return this.getStreamId() + this.siteId
}
getStreamId(): string | undefined {
if (!this.channelId) return undefined
if (!this.feedId) return this.channelId
return `${this.channelId}@${this.feedId}`
return generateXMLTV({
channels: this.channels.all(),
programs: this.programs.all(),
date: currDate.toJSON()
})
}
}
+59
View File
@@ -0,0 +1,59 @@
import { Dictionary } from '@freearhey/core'
import epgGrabber from 'epg-grabber'
import { Feed, Channel } from '.'
export class GuideChannel {
channelId?: string
channel?: Channel
feedId?: string
feed?: Feed
xmltvId?: string
languageCode?: string
siteId?: string
logoUrl?: string
siteDomain?: string
siteName?: string
constructor(data: epgGrabber.Channel) {
const [channelId, feedId] = data.xmltv_id ? data.xmltv_id.split('@') : [undefined, undefined]
this.channelId = channelId
this.feedId = feedId
this.xmltvId = data.xmltv_id
this.languageCode = data.lang
this.siteId = data.site_id
this.logoUrl = data.logo
this.siteDomain = data.site
this.siteName = data.name
}
withChannel(channelsKeyById: Dictionary): this {
if (this.channelId) this.channel = channelsKeyById.get(this.channelId)
return this
}
withFeed(feedsKeyByStreamId: Dictionary): this {
if (this.feedId) this.feed = feedsKeyByStreamId.get(this.getStreamId())
return this
}
getStreamId(): string {
if (!this.channelId) return ''
if (!this.feedId) return this.channelId
return `${this.channelId}@${this.feedId}`
}
toJSON() {
return {
channel: this.channelId || null,
feed: this.feedId || null,
site: this.siteDomain || '',
site_id: this.siteId || '',
site_name: this.siteName || '',
lang: this.languageCode || ''
}
}
}
+6 -3
View File
@@ -1,6 +1,9 @@
export * from './issue'
export * from './site'
export * from './channel'
export * from './feed'
export * from './stream'
export * from './guide'
export * from './guideChannel'
export * from './issue'
export * from './logo'
export * from './site'
export * from './stream'
export * from './channelList'
+41
View File
@@ -0,0 +1,41 @@
import { Collection, type Dictionary } from '@freearhey/core'
import type { LogoData } from '../types/logo'
import { type Feed } from './feed'
export class Logo {
channelId?: string
feedId?: string
feed?: Feed
tags: Collection = new Collection()
width: number = 0
height: number = 0
format?: string
url?: string
constructor(data?: LogoData) {
if (!data) return
this.channelId = data.channel
this.feedId = data.feed || undefined
this.tags = new Collection(data.tags)
this.width = data.width
this.height = data.height
this.format = data.format || undefined
this.url = data.url
}
withFeed(feedsKeyByStreamId: Dictionary): this {
if (!this.feedId) return this
this.feed = feedsKeyByStreamId.get(this.getStreamId())
return this
}
getStreamId(): string {
if (!this.channelId) return ''
if (!this.feedId) return this.channelId
return `${this.channelId}@${this.feedId}`
}
}
-1
View File
@@ -15,7 +15,6 @@ export type ChannelData = {
closed: string
replaced_by: string
website: string
logo: string
}
export type ChannelSearchableData = {
+1
View File
@@ -16,4 +16,5 @@ export type DataLoaderData = {
timezones: object | object[]
guides: object | object[]
streams: object | object[]
logos: object | object[]
}
+6 -2
View File
@@ -1,12 +1,16 @@
import { Collection, Dictionary } from '@freearhey/core'
export type DataProcessorData = {
guideChannelsGroupedByStreamId: Dictionary
feedsGroupedByChannelId: Dictionary
guidesGroupedByStreamId: Dictionary
logosGroupedByChannelId: Dictionary
logosGroupedByStreamId: Dictionary
feedsKeyByStreamId: Dictionary
streamsGroupedById: Dictionary
channelsKeyById: Dictionary
guideChannels: Collection
channels: Collection
streams: Collection
guides: Collection
feeds: Collection
logos: Collection
}
+9
View File
@@ -0,0 +1,9 @@
export type LogoData = {
channel: string
feed: string | null
tags: string[]
width: number
height: number
format: string | null
url: string
}
+3 -3
View File
@@ -107,19 +107,19 @@ https://epgshare01.online/epgshare01/
Windows (Command Prompt):
```sh
SET "NODE_OPTIONS=--max-old-space-size=5000" && npm run grab --- --channels=sites/epgshare01.online/epgshare01.online_<TAG>.channels.xml
SET "NODE_OPTIONS=--max-old-space-size=6000" && npm run grab --- --channels=sites/epgshare01.online/epgshare01.online_<TAG>.channels.xml
```
Windows (PowerShell):
```sh
$env:NODE_OPTIONS="--max-old-space-size=5000"; npm run grab --- --channels=sites/epgshare01.online/epgshare01.online_<TAG>.channels.xml
$env:NODE_OPTIONS="--max-old-space-size=6000"; npm run grab --- --channels=sites/epgshare01.online/epgshare01.online_<TAG>.channels.xml
```
Linux and macOS:
```sh
NODE_OPTIONS=--max-old-space-size=5000 npm run grab --- --channels=sites/epgshare01.online/epgshare01.online_<TAG>.channels.xml
NODE_OPTIONS=--max-old-space-size=6000 npm run grab --- --channels=sites/epgshare01.online/epgshare01.online_<TAG>.channels.xml
```
### Update channel list
+1 -1
View File
@@ -521,7 +521,7 @@
<channel site="i.mjh.nz" lang="en" xmltv_id="" site_id="SamsungTVPlus/gb#GBBA33000362J">Pluto TV Paranormal</channel>
<channel site="i.mjh.nz" lang="en" xmltv_id="" site_id="SamsungTVPlus/gb#GBBA33000529G">Tennis Channel International</channel>
<channel site="i.mjh.nz" lang="en" xmltv_id="" site_id="SamsungTVPlus/gb#GBBA33000540B">PBS History</channel>
<channel site="i.mjh.nz" lang="en" xmltv_id="" site_id="SamsungTVPlus/gb#GBBA33000557H">MovieSphere</channel>
<channel site="i.mjh.nz" lang="en" xmltv_id="SamsungMovieSphere.us" site_id="SamsungTVPlus/gb#GBBA33000557H">MovieSphere</channel>
<channel site="i.mjh.nz" lang="en" xmltv_id="" site_id="SamsungTVPlus/gb#GBBA220000751">INWILD</channel>
<channel site="i.mjh.nz" lang="en" xmltv_id="" site_id="SamsungTVPlus/gb#GBBA330001986">Gusto TV</channel>
<channel site="i.mjh.nz" lang="en" xmltv_id="" site_id="SamsungTVPlus/gb#GBBA330002536">Beano TV</channel>
+1
View File
@@ -0,0 +1 @@
{"api_key":"eyJhbGciOiJIUzI1NiJ9.eyJkYXRhIjp7InR5cGUiOiJhcGlrZXkifSwiZXhwIjoxNzUxNDc4MDQ2fQ.kxcPR76NBdQYIrsUnmBpqzcSmPD-H7b_-I_SjFi1zY4","api_key_expires_at":"2025-07-03T00:40:46+07:00"}
File diff suppressed because one or more lines are too long
+1
View File
@@ -0,0 +1 @@
{"errors":[{"detail":"Livestreaming not found"}]}
+15 -5
View File
@@ -2,12 +2,14 @@
<channels>
<channel site="vidio.com" lang="id" xmltv_id="ABCAustralia.au" site_id="7150">ABC Australia</channel>
<channel site="vidio.com" lang="id" xmltv_id="AfricanewsEnglish.fr" site_id="12784">AFRICANEWS TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="AjwaTV.id" site_id="7464">Ajwa TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="AlJazeera.qa@English" site_id="6410">Aljazeera</channel>
<channel site="vidio.com" lang="id" xmltv_id="ANTV.id" site_id="782">ANTV</channel>
<channel site="vidio.com" lang="id" xmltv_id="ArirangWorld.kr" site_id="6784">Arirang</channel>
<channel site="vidio.com" lang="id" xmltv_id="beINSports1.qa@Indonesia" site_id="6299">Bein 1</channel>
<channel site="vidio.com" lang="id" xmltv_id="" site_id="17875">Bein 2</channel>
<channel site="vidio.com" lang="id" xmltv_id="beINSports3.qa@Indonesia" site_id="6317">Bein 3</channel>
<channel site="vidio.com" lang="id" xmltv_id="BeritaSatuEnglish.id" site_id="18280">BeritaSatu</channel>
<channel site="vidio.com" lang="id" xmltv_id="BTV.id" site_id="6165">BTV</channel>
<channel site="vidio.com" lang="id" xmltv_id="ChampionsTV1.id" site_id="6685">CTV 1</channel>
<channel site="vidio.com" lang="id" xmltv_id="ChampionsTV2.id" site_id="6686">CTV 2</channel>
@@ -15,36 +17,44 @@
<channel site="vidio.com" lang="id" xmltv_id="ChampionsTV5.id" site_id="9182">CTV 5</channel>
<channel site="vidio.com" lang="id" xmltv_id="ChampionsTV6.id" site_id="9183">CTV 6</channel>
<channel site="vidio.com" lang="id" xmltv_id="ChampionsTVEPL.id" site_id="9353">Premier League TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="" site_id="18189">Champions Golf 1</channel>
<channel site="vidio.com" lang="id" xmltv_id="" site_id="18190">Champions Golf 2</channel>
<channel site="vidio.com" lang="id" xmltv_id="CNA.sg" site_id="6411">News Asia</channel>
<channel site="vidio.com" lang="id" xmltv_id="DAAITV.id" site_id="6482">DAAI TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="DaystarTV.us" site_id="18622">Daystar TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="DW.de@English" site_id="5075">DW English</channel>
<channel site="vidio.com" lang="id" xmltv_id="ElshintaTV.id" site_id="10975">Elshinta TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="EuronewsEnglish.fr" site_id="6412">Euro News</channel>
<channel site="vidio.com" lang="id" xmltv_id="" site_id="18105">GGS TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="HipHipHoree.id" site_id="7052">Hip Hip Horee!</channel>
<channel site="vidio.com" lang="id" xmltv_id="Horee.id" site_id="6397">Horee</channel>
<channel site="vidio.com" lang="id" xmltv_id="Indosiar.id" site_id="205">Indosiar</channel>
<channel site="vidio.com" lang="id" xmltv_id="JakTV.id" site_id="5415">Jaktv</channel>
<channel site="vidio.com" lang="id" xmltv_id="JPMTV.id" site_id="9714">jawaposTV</channel>
<channel site="vidio.com" lang="id" xmltv_id="JTV.id" site_id="9713">JTV</channel>
<channel site="vidio.com" lang="id" xmltv_id="KompasTV.id" site_id="874">Kompas TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="MagnaChannel.id" site_id="7230">Magna TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="MetroGlobeNetwork.id" site_id="8815">Metro Globe Network</channel>
<channel site="vidio.com" lang="id" xmltv_id="MakkahTV.sa" site_id="6852">Makkah TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="NET.id" site_id="875">MDTV</channel>
<channel site="vidio.com" lang="id" xmltv_id="MetroTV.id" site_id="777">Metro TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="Moji.id" site_id="206">Moji</channel>
<channel site="vidio.com" lang="id" xmltv_id="" site_id="7619">MUSICA</channel>
<channel site="vidio.com" lang="id" xmltv_id="NBATV.us" site_id="6717">NBA TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="NET.id" site_id="875">NET TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="NHKWorldJapan.jp" site_id="7968">NHK World Japan</channel>
<channel site="vidio.com" lang="id" xmltv_id="NusantaraTV.id" site_id="7432">Nusantara TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="RajawaliTV.id" site_id="1561">RTV</channel>
<channel site="vidio.com" lang="id" xmltv_id="RANSEntertainment.id" site_id="8241">RANS Channel</channel>
<channel site="vidio.com" lang="id" xmltv_id="ROCKEntertainment.sg" site_id="8120">ROCK Entertainment</channel>
<channel site="vidio.com" lang="id" xmltv_id="ROCKExtreme.sg" site_id="8121">Rock Action</channel>
<channel site="vidio.com" lang="id" xmltv_id="SCTV.id" site_id="204">SCTV</channel>
<channel site="vidio.com" lang="id" xmltv_id="SEAToday.id" site_id="7687">SEA TODAY</channel>
<channel site="vidio.com" lang="id" xmltv_id="SPOTV2.id" site_id="17140">SPOTV 2</channel>
<channel site="vidio.com" lang="id" xmltv_id="SPOTV.id" site_id="17139">SPOTV</channel>
<channel site="vidio.com" lang="id" xmltv_id="TawafTV.id" site_id="12607">Tawaf TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="Trans7.id" site_id="734">Trans7</channel>
<channel site="vidio.com" lang="id" xmltv_id="TransTV.id" site_id="733">TRANS TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="TV5MondeAsia.fr" site_id="17278">TV5Monde</channel>
<channel site="vidio.com" lang="id" xmltv_id="tvNAsia.hk" site_id="6362">TVN</channel>
<channel site="vidio.com" lang="id" xmltv_id="tvOne.id" site_id="783">TVOne</channel>
<channel site="vidio.com" lang="id" xmltv_id="TVRINasional.id" site_id="6441">TVRI</channel>
<channel site="vidio.com" lang="id" xmltv_id="TVRI.id" site_id="6441">TVRI</channel>
<channel site="vidio.com" lang="id" xmltv_id="UChannel.id" site_id="6898">U-Channel TV</channel>
<channel site="vidio.com" lang="id" xmltv_id="ZooMoo.sg" site_id="6533">Zoomoo</channel>
</channels>
+58 -81
View File
@@ -1,112 +1,89 @@
const cheerio = require('cheerio')
const axios = require('axios')
const dayjs = require('dayjs')
const utc = require('dayjs/plugin/utc')
const timezone = require('dayjs/plugin/timezone')
const customParseFormat = require('dayjs/plugin/customParseFormat')
const crypto = require('crypto')
dayjs.extend(utc)
dayjs.extend(timezone)
dayjs.extend(customParseFormat)
const tz = 'Asia/Jakarta'
const WEB_CLIENT_SECRET = Buffer.from('dPr0QImQ7bc5o9LMntNba2DOsSbZcjUh')
const WEB_CLIENT_IV = Buffer.from('C8RWsrtFsoeyCyPt')
module.exports = {
site: 'vidio.com',
days: 2,
url({ channel }) {
return `https://www.vidio.com/live/${channel.site_id}/schedules`
url({ date, channel }) {
return `https://api.vidio.com/livestreamings/${channel.site_id}/schedules?filter[date]=${date.format('YYYY-MM-DD')}`
},
parser({ content, date }) {
request: {
async headers() {
const session = await loadSessionDetails()
if (!session || !session.api_key) return null
var cipher = crypto.createCipheriv('aes-256-cbc', WEB_CLIENT_SECRET, WEB_CLIENT_IV)
return {
'X-API-Key': cipher.update(session.api_key, 'utf8', 'base64') + cipher.final('base64'),
'X-Secure-Level': 2
}
}
},
parser({ content }) {
const programs = []
const items = parseItems(content, date)
items.forEach(item => {
const prev = programs[programs.length - 1]
const $item = cheerio.load(item)
let start = parseStart($item, date)
if (prev && start < prev.start) {
start = start.add(1, 'd')
date = date.add(1, 'd')
const json = JSON.parse(content)
if (Array.isArray(json?.data)) {
for (const program of json.data) {
programs.push({
title: program.attributes.title,
description: program.attributes.description,
start: dayjs(program.attributes.start_time),
stop: dayjs(program.attributes.end_time),
image: program.attributes.image_landscape_url
})
}
let stop = parseStop($item, date)
if (stop < start) {
stop = stop.add(1, 'd')
date = date.add(1, 'd')
}
programs.push({
title: parseTitle($item),
start,
stop
})
})
}
return programs
},
async channels() {
const axios = require('axios')
const cheerio = require('cheerio')
const result = await axios
.get('https://www.vidio.com/categories/daftar-channel-tv-radio-live-sports')
const channels = []
const json = await axios
.get(
'https://api.vidio.com/livestreamings?stream_type=tv_stream',
{
headers: await this.request.headers()
}
)
.then(response => response.data)
.catch(console.error)
const $ = cheerio.load(result)
const itemGroups = $('div[data-variation="circle_horizontal"] ul').toArray()
const channels = []
itemGroups.forEach(group => {
const $group = $(group)
let skip = false
const sites = []
const items = $group.find('a[data-testid="circle-card"]').toArray()
items.forEach(item => {
const name = $(item).find('span[data-testid="circle-title"]').text()
// skip radio channels
if (name.toLowerCase().indexOf('fm') >= 0 || name.toLowerCase().indexOf('radio') >= 0) {
skip = true
return true
}
let url = $(item).attr('href')
url = url.substr(url.lastIndexOf('/') + 1)
const matches = url.match(/(\d+)/)
sites.push({
if (Array.isArray(json?.data)) {
for (const channel of json.data) {
channels.push({
lang: 'id',
site_id: matches[0],
name: name
site_id: channel.id,
name: channel.attributes.title
})
})
if (!skip && sites.length) {
channels.push(...sites)
}
})
}
return channels
}
}
function parseStart($item, date) {
const timeString = $item('div.b-livestreaming-daily-schedule__item-content-caption').text()
const [, start] = timeString.match(/(\d{2}:\d{2}) -/) || [null, null]
return dayjs.tz(`${date.format('YYYY-MM-DD')} ${start}`, 'YYYY-MM-DD HH:mm', tz)
}
function parseStop($item, date) {
const timeString = $item('div.b-livestreaming-daily-schedule__item-content-caption').text()
const [, stop] = timeString.match(/- (\d{2}:\d{2}) WIB/) || [null, null]
return dayjs.tz(`${date.format('YYYY-MM-DD')} ${stop}`, 'YYYY-MM-DD HH:mm', tz)
}
function parseTitle($item) {
return $item('div.b-livestreaming-daily-schedule__item-content-title').text()
}
function parseItems(content, date) {
const $ = cheerio.load(content)
return $(
`#schedule-content-${date.format(
'YYYYMMDD'
)} > .b-livestreaming-daily-schedule__scroll-container .b-livestreaming-daily-schedule__item`
).toArray()
}
function loadSessionDetails() {
return axios
.post(
'https://www.vidio.com/auth',
{},
{
headers: {
'Content-Type': 'application/json'
}
}
)
.then(r => r.data)
.catch(console.log)
}
+45 -31
View File
@@ -1,53 +1,67 @@
const { parser, url } = require('./vidio.com.config.js')
const { parser, url, request } = require('./vidio.com.config.js')
const fs = require('fs')
const path = require('path')
const axios = require('axios')
const dayjs = require('dayjs')
const utc = require('dayjs/plugin/utc')
const customParseFormat = require('dayjs/plugin/customParseFormat')
dayjs.extend(customParseFormat)
dayjs.extend(utc)
const date = dayjs.utc('2021-11-24', 'YYYY-MM-DD').startOf('d')
jest.mock('axios')
const date = dayjs.utc('2025-07-01', 'YYYY-MM-DD').startOf('d')
const channel = {
site_id: '7464',
xmltv_id: 'AjwaTV.id'
site_id: '204',
xmltv_id: 'SCTV.id'
}
it('can generate valid url', () => {
expect(url({ channel })).toBe('https://www.vidio.com/live/7464/schedules')
expect(url({ channel, date })).toBe(
'https://api.vidio.com/livestreamings/204/schedules?filter[date]=2025-07-01'
)
})
it('can generate valid request headers', async () => {
axios.post.mockImplementation(url => {
if (url === 'https://www.vidio.com/auth') {
return Promise.resolve({
data: JSON.parse(fs.readFileSync(path.resolve(__dirname, '__data__/auth.json')))
})
} else {
return Promise.resolve({ data: '' })
}
})
const result = await request.headers()
expect(result).toMatchObject({
'X-API-Key':
'CH1ZFsN4N/MIfAds1DL9mP151CNqIpWHqZGRr+LkvUyiq3FRPuP1Kt6aK+pG3nEC1FXt0ZAAJ5FKP8QU8CZ5/jQdSYLVeFwl9NoIkegVpR6b7W2ZwbaF00OPr6ON1/FpLQ3RiUzTPpAqe7f+fwhOr0KrKy8PpCa54OHogaEjI3w=',
'X-Secure-Level': 2,
})
})
it('can parse response', () => {
const content =
'<div class="b-livestreaming-daily-schedule__section b-livestreaming-daily-schedule__contents"> <div class="b-livestreaming-daily-schedule__content" id="schedule-content-20211124"> <div class="js-ahoy b-livestreaming-schedule__ahoy-impression" data-ahoy-component="Ahoy::Builder" data-ahoy-impression="true" data-ahoy-title="LIVESTREAMING" data-use-bounding-client-rect="true" ></div><div class="b-livestreaming-daily-schedule__scroll-container"> <a class="" href="/watch/2361692-30-hari-30-juz-24-november-2021" ><div class="b-livestreaming-daily-schedule__item js-ahoy" data-ahoy-click="true" data-ahoy-component="Ahoy::Builder" data-ahoy-props=\'{"section":"schedule","stream_type":"TvStream","livestreaming_id":7464,"schedule_day":-1,"schedule_id":1471806,"schedule_title":"30 Hari 30 Juz","action":"click"}\' data-ahoy-title="LIVESTREAMING" id="schedule-item-1471806" > <div class="b-livestreaming-daily-schedule__item-icon"> <div class="b-livestreaming-daily-schedule__item-icon-play"></div></div><div class="b-livestreaming-daily-schedule__item-content"> <div class="b-livestreaming-daily-schedule__item-content-caption"> 00:30 - 01:30 WIB </div><div class="b-livestreaming-daily-schedule__item-content-title">30 Hari 30 Juz</div></div></div></a ><a class="" href="/watch/2361785-makkah-live-24-november-2021" ><div class="b-livestreaming-daily-schedule__item js-ahoy" data-ahoy-click="true" data-ahoy-component="Ahoy::Builder" data-ahoy-props=\'{"section":"schedule","stream_type":"TvStream","livestreaming_id":7464,"schedule_day":-1,"schedule_id":1471807,"schedule_title":"Makkah Live","action":"click"}\' data-ahoy-title="LIVESTREAMING" id="schedule-item-1471807" > <div class="b-livestreaming-daily-schedule__item-icon"> <div class="b-livestreaming-daily-schedule__item-icon-play"></div></div><div class="b-livestreaming-daily-schedule__item-content"> <div class="b-livestreaming-daily-schedule__item-content-caption"> 01:30 - 04:00 WIB </div><div class="b-livestreaming-daily-schedule__item-content-title">Makkah Live</div></div></div></a ><a class="" href="/watch/2362889-ftv-islami-24-november-2021" ><div class="b-livestreaming-daily-schedule__item js-ahoy" data-ahoy-click="true" data-ahoy-component="Ahoy::Builder" data-ahoy-props=\'{"section":"schedule","stream_type":"TvStream","livestreaming_id":7464,"schedule_day":-1,"schedule_id":1473829,"schedule_title":"FTV Islami","action":"click"}\' data-ahoy-title="LIVESTREAMING" id="schedule-item-1473829" > <div class="b-livestreaming-daily-schedule__item-icon"> <div class="b-livestreaming-daily-schedule__item-icon-play"></div></div><div class="b-livestreaming-daily-schedule__item-content"> <div class="b-livestreaming-daily-schedule__item-content-caption"> 22:30 - 00:30 WIB </div><div class="b-livestreaming-daily-schedule__item-content-title">FTV Islami</div></div></div></a > </div></div><div class="b-livestreaming-daily-schedule__content tab-active" id="schedule-content-20211125"> <div class="js-ahoy b-livestreaming-schedule__ahoy-impression" data-ahoy-component="Ahoy::Builder" data-ahoy-impression="true" data-ahoy-title="LIVESTREAMING" data-use-bounding-client-rect="true" ></div><div class="b-livestreaming-daily-schedule__scroll-container"> <a class="" href="/watch/2362921-30-hari-30-juz-25-november-2021" ><div class="b-livestreaming-daily-schedule__item js-ahoy" data-ahoy-click="true" data-ahoy-component="Ahoy::Builder" data-ahoy-props=\'{"section":"schedule","stream_type":"TvStream","livestreaming_id":7464,"schedule_day":0,"schedule_id":1473830,"schedule_title":"30 Hari 30 Juz","action":"click"}\' data-ahoy-title="LIVESTREAMING" id="schedule-item-1473830" > <div class="b-livestreaming-daily-schedule__item-icon"> <div class="b-livestreaming-daily-schedule__item-icon-play"></div></div><div class="b-livestreaming-daily-schedule__item-content"> <div class="b-livestreaming-daily-schedule__item-content-caption"> 00:30 - 01:30 WIB </div><div class="b-livestreaming-daily-schedule__item-content-title">30 Hari 30 Juz</div></div></div></a > </div></div></div>'
const result = parser({ content, date }).map(p => {
const content = fs.readFileSync(path.resolve(__dirname, '__data__/content.json'))
const results = parser({ content, channel, date }).map(p => {
p.start = p.start.toJSON()
p.stop = p.stop.toJSON()
return p
})
expect(result).toMatchObject([
{
start: '2021-11-23T17:30:00.000Z',
stop: '2021-11-23T18:30:00.000Z',
title: '30 Hari 30 Juz'
},
{
start: '2021-11-23T18:30:00.000Z',
stop: '2021-11-23T21:00:00.000Z',
title: 'Makkah Live'
},
{
start: '2021-11-24T15:30:00.000Z',
stop: '2021-11-24T17:30:00.000Z',
title: 'FTV Islami'
}
])
expect(results.length).toBe(21)
expect(results[0]).toMatchObject({
start: '2025-06-30T15:57:00.000Z',
stop: '2025-06-30T17:29:00.000Z',
title: 'Ftv PrimeTime : Cinta Dodol Inilah Yang Membuatku Lengket Padamu',
description: 'Film televisi yang mengangkat kisah romantisme kehidupan dengan konflik yang menarik. tayang setiap hari'
})
})
it('can handle empty guide', () => {
const result = parser({
date,
channel,
content: '<!DOCTYPE html><html><head></head><body></body></html>'
})
expect(result).toMatchObject([])
const content = fs.readFileSync(path.resolve(__dirname, '__data__/no_content.json'))
const results = parser({ content, channel })
expect(results).toMatchObject([])
})
@@ -1,8 +1,9 @@
<?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url></channel>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
</tv>
@@ -1,14 +1,15 @@
<?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
<channel id="Channel1.us"><display-name>Custom Channel 1</display-name><url>https://example.com</url></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example2.com</url></channel>
<channel id="Channel2.us"><display-name>Custom Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url></channel>
<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example2.com</url></channel>
<channel id="Channel4.us"><display-name>Channel 4</display-name><url>https://example2.com</url></channel>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
<channel id="Channel3.us@Wrong"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example2.com</url></channel>
<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/><url>https://example2.com</url></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example2.com</url></channel>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221019044000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example2.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="en">Program1 (example2.com)</title></programme>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel4.us"><title lang="en">Program1 (example2.com)</title></programme>
<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel3.us@Wrong"><title lang="en">Program1 (example2.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel4.us@HD"><title lang="en">Program1 (example2.com)</title></programme>
</tv>
Binary file not shown.
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url></channel>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
</tv>
@@ -3,6 +3,6 @@
<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example.com</url></channel>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
</tv>
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
</tv>
@@ -1,14 +1,15 @@
<?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url><lcn>36</lcn></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example2.com</url></channel>
<channel id="Channel2.us"><display-name>Channel 2</display-name><icon src="https://i.imgur.com/qmRnD0M.png"/><url>https://example.com</url></channel>
<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example2.com</url></channel>
<channel id="Channel4.us"><display-name>Channel 4</display-name><url>https://example2.com</url></channel>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<channel id="Channel4.us@HD"><display-name>Channel 4</display-name><icon src="https://i.imgur.com/BPzH88J.png"/><url>https://example2.com</url></channel>
<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example2.com</url></channel>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221019044000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example2.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="en">Program1 (example2.com)</title></programme>
<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel4.us"><title lang="en">Program1 (example2.com)</title></programme>
<programme start="20221020043100 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="en">Program1 (example.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="en">Program1 (example2.com)</title></programme>
<programme start="20221019043100 +0000" stop="20221019071000 +0000" channel="Channel4.us@HD"><title lang="en">Program1 (example2.com)</title></programme>
</tv>
+9 -14
View File
@@ -9,8 +9,7 @@
"categories": [],
"is_nsfw": false,
"closed": "2020-01-01",
"replaced_by": "R6.co",
"logo": "https://www.directv.com/images/logos/channels/dark/large/579.png"
"replaced_by": "R6.co"
},
{
"id": "Bravos.us",
@@ -20,8 +19,7 @@
"subdivision": null,
"city": null,
"categories": [],
"is_nsfw": false,
"logo": "https://www.directv.com/images/logos/channels/dark/large/579.png"
"is_nsfw": false
},
{
"id": "CNNInternational.us",
@@ -34,8 +32,7 @@
"categories": [
"news"
],
"is_nsfw": false,
"logo": "https://i.imgur.com/2BXCg0x.jpg"
"is_nsfw": false
},
{
"id": "MNetMovies2.za",
@@ -45,11 +42,10 @@
"subdivision": null,
"city": null,
"categories": [],
"is_nsfw": false,
"logo": "https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png"
"is_nsfw": false
},
{"id":"6eren.dk","name":"6'eren","alt_names":[],"network":null,"owners":["Warner Bros. Discovery EMEA"],"country":"DK","subdivision":null,"city":null,"broadcast_area":["c/DK"],"languages":["dan"],"categories":[],"is_nsfw":false,"launched":"2009-01-01","closed":null,"replaced_by":null,"website":"http://www.6-eren.dk/","logo":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"},
{"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/","logo":"https://i.imgur.com/rPzH88J.png"},
{"id":"6eren.dk","name":"6'eren","alt_names":[],"network":null,"owners":["Warner Bros. Discovery EMEA"],"country":"DK","subdivision":null,"city":null,"broadcast_area":["c/DK"],"languages":["dan"],"categories":[],"is_nsfw":false,"launched":"2009-01-01","closed":null,"replaced_by":null,"website":"http://www.6-eren.dk/"},
{"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/"},
{
"id": "CNN.us",
"name": "CNN",
@@ -58,9 +54,8 @@
"subdivision": null,
"city": null,
"categories": [],
"is_nsfw": false,
"logo": "https://www.directv.com/images/logos/channels/dark/large/579.png"
"is_nsfw": false
},
{"id":"Channel2.us","name":"Channel 2 [API]","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/US"],"languages":["eng"],"categories":[],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"","logo":"https://i.imgur.com/rPzH88J.png"},
{"id":"Channel3.us","name":"Channel 3 [API]","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/US"],"languages":["eng"],"categories":[],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"","logo":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"}
{"id":"Channel2.us","name":"Channel 2 [API]","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/US"],"languages":["eng"],"categories":[],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":""},
{"id":"Channel3.us","name":"Channel 3 [API]","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/US"],"languages":["eng"],"categories":[],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":""}
]
+28
View File
@@ -26,5 +26,33 @@
],
"languages": [],
"video_format": "480i"
},
{
"channel": "Bravo.us",
"id": "East",
"name": "East",
"is_main": true,
"broadcast_area": [
"r/EUR"
],
"timezones": [
"America/New_York"
],
"languages": [],
"video_format": "480i"
},
{
"channel": "Channel4.us",
"id": "HD",
"name": "HD",
"is_main": true,
"broadcast_area": [
"r/EUR"
],
"timezones": [
"America/New_York"
],
"languages": [],
"video_format": "480i"
}
]
+4
View File
@@ -0,0 +1,4 @@
[
{"channel":"Channel3.us","feed":null,"tags":[],"width":334,"height":210,"format":"PNG","url":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"},
{"channel":"Channel4.us","feed":"HD","tags":[],"width":334,"height":210,"format":"PNG","url":"https://i.imgur.com/BPzH88J.png"}
]
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<channels>
<channel site="example.com" lang="en" xmltv_id="Bravo.us@East" site_id="140#Tes't">Bravo's</channel>
<channel site="example.com" lang="en" xmltv_id="Bravo.us@East" site_id="140#Tes't" lcn="36">Bravo's</channel>
</channels>
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<channels site="example.com">
<channel lang="en" xmltv_id="CNNInternational" site_id="140">CNN International</channel>
<channel lang="en" xmltv_id="Bravo.us@East" site_id="150">Bravo</channel>
<channel lang="en" xmltv_id="Bravo.us@West" site_id="151">Bravo</channel>
<channel lang="en" xmltv_id="Bravo.us" site_id="150">Bravo</channel>
<channel lang="en" xmltv_id="Bravo.us" site_id="151">Bravo</channel>
</channels>
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<channels site="example.com">
<channel lang="en" xmltv_id="Bravo.us@West" site_id="150">Bravo</channel>
</channels>
@@ -3,7 +3,7 @@
<channel site="example.com" lang="en" xmltv_id="Channel1.us" site_id="140">Custom Channel 1</channel>
<channel site="example.com" lang="en" xmltv_id="Channel2.us" site_id="142" logo="https://i.imgur.com/qmRnD0M.png">Custom Channel 2</channel>
<channel site="example.com" lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
<channel site="example2.com" lang="en" xmltv_id="Channel3.us" site_id="150">Channel 3</channel>
<channel site="example2.com" lang="en" xmltv_id="Channel4.us" site_id="152">Channel 4</channel>
<channel site="example2.com" lang="en" xmltv_id="Channel3.us@Wrong" site_id="150">Channel 3</channel>
<channel site="example2.com" lang="en" xmltv_id="Channel4.us@HD" site_id="152">Channel 4</channel>
<channel site="example2.com" lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
</channels>
@@ -20,7 +20,7 @@ module.exports = {
return [
{
title: 'Program1 (example.com)',
start: `${date.format('YYYY-MM-DD')}T04:30:00.000Z`,
start: `${date.format('YYYY-MM-DD')}T04:31:00.000Z`,
stop: `${date.format('YYYY-MM-DD')}T07:10:00.000Z`
}
]
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<channels site="example.com">
<channel lang="en" xmltv_id="Channel2.us" site_id="142" logo="https://i.imgur.com/qmRnD0M.png" lcn="36">Channel 2</channel>
<channel lang="en" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
<channel lang="en" xmltv_id="Channel2.us" site_id="142" logo="https://i.imgur.com/qmRnD0M.png">Channel 2</channel>
<channel lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
</channels>
@@ -20,7 +20,7 @@ module.exports = {
return [
{
title: 'Program1 (example.com)',
start: `${date.format('YYYY-MM-DD')}T04:30:00.000Z`,
start: `${date.format('YYYY-MM-DD')}T04:31:00.000Z`,
stop: `${date.format('YYYY-MM-DD')}T07:10:00.000Z`
}
]
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<channels site="example2.com">
<channel lang="en" xmltv_id="Channel3.us" site_id="150">Channel 3</channel>
<channel lang="en" xmltv_id="Channel4.us" site_id="152">Channel 4</channel>
<channel lang="en" xmltv_id="Channel4.us@HD" site_id="152">Channel 4</channel>
<channel lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
</channels>
@@ -15,7 +15,7 @@ module.exports = {
return [
{
title: 'Program1 (example2.com)',
start: `${date.format('YYYY-MM-DD')}T04:30:00.000Z`,
start: `${date.format('YYYY-MM-DD')}T04:31:00.000Z`,
stop: `${date.format('YYYY-MM-DD')}T07:10:00.000Z`
}
]
+27 -7
View File
@@ -28,20 +28,40 @@ describe('channels:validate', () => {
}
})
it('will show a message if the file contains a channel with wrong xmltv_id', () => {
it('will show a message if the file contains a channel with wrong channel id', () => {
try {
const cmd = `${ENV_VAR} npm run channels:validate --- tests/__data__/input/channels_validate/wrong_xmltv_id.channels.xml`
const cmd = `${ENV_VAR} npm run channels:validate --- tests/__data__/input/channels_validate/wrong_channel_id.channels.xml`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
process.exit(1)
} catch (error) {
expect((error as ExecError).status).toBe(1)
expect((error as ExecError).stdout).toContain(`
┌─────────┬──────────────────┬──────┬────────────────────┬─────────┬─────────────────────┐
│ (index) │ type │ lang │ xmltv_id │ site_id │ name │
├─────────┼──────────────────┼──────┼────────────────────┼─────────┼─────────────────────┤
│ 0 │ 'wrong_xmltv_id' │ 'en' │ 'CNNInternational' │ '140' │ 'CNN International' │
└─────────┴──────────────────┴──────┴────────────────────┴─────────┴─────────────────────┘
┌─────────┬────────────────────┬──────┬────────────────────┬─────────┬─────────────────────┐
│ (index) │ type │ lang │ xmltv_id │ site_id │ name │
├─────────┼────────────────────┼──────┼────────────────────┼─────────┼─────────────────────┤
│ 0 │ 'wrong_channel_id' │ 'en' │ 'CNNInternational' │ '140' │ 'CNN International' │
└─────────┴────────────────────┴──────┴────────────────────┴─────────┴─────────────────────┘
1 error(s) in 1 file(s)
`)
}
})
it('will show a message if the file contains a channel with wrong feed id', () => {
try {
const cmd = `${ENV_VAR} npm run channels:validate --- tests/__data__/input/channels_validate/wrong_feed_id.channels.xml`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
process.exit(1)
} catch (error) {
expect((error as ExecError).status).toBe(1)
expect((error as ExecError).stdout).toContain(`
┌─────────┬─────────────────┬──────┬─────────────────┬─────────┬─────────┐
│ (index) │ type │ lang │ xmltv_id │ site_id │ name │
├─────────┼─────────────────┼──────┼─────────────────┼─────────┼─────────┤
│ 0 │ 'wrong_feed_id' │ 'en' │ 'Bravo.us@West' │ '150' │ 'Bravo' │
└─────────┴─────────────────┴──────┴─────────────────┴─────────┴─────────┘
1 error(s) in 1 file(s)
`)
+91 -80
View File
@@ -4,7 +4,8 @@ import { Zip } from '@freearhey/core'
import fs from 'fs-extra'
import path from 'path'
const ENV_VAR = 'cross-env SITES_DIR=tests/__data__/input/epg_grab/sites CURR_DATE=2022-10-20 DATA_DIR=tests/__data__/input/__data__'
const ENV_VAR =
'cross-env SITES_DIR=tests/__data__/input/epg_grab/sites CURR_DATE=2022-10-20 DATA_DIR=tests/__data__/input/__data__'
beforeEach(() => {
fs.emptyDirSync('tests/__data__/output')
@@ -14,85 +15,12 @@ describe('epg:grab', () => {
it('can grab epg by site name', () => {
const cmd = `${ENV_VAR} npm run grab --- --site=example.com --output="${path.resolve(
'tests/__data__/output/guide.xml'
)}"`
)}" --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guide.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guide_2.xml')
)
})
it('can grab epg with multiple channels.xml files', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/**/*.channels.xml --output=tests/__data__/output/guide.xml`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guide.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guide.xml')
)
})
it('can grab epg with gzip option enabled', async () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/**/*.channels.xml --output="${path.resolve(
'tests/__data__/output/guide.xml'
)}" --gzip`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guide.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guide.xml')
)
const zip = new Zip()
const expected = zip.decompress(fs.readFileSync('tests/__data__/output/guide.xml.gz'))
const result = zip.decompress(
fs.readFileSync('tests/__data__/expected/epg_grab/guide.xml.gz')
)
expect(expected).toEqual(result)
}, 30000)
it('can grab epg with wildcard as output', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels="tests/__data__/input/epg_grab/sites/example.com/example.com.channels.xml" --output="tests/__data__/output/guides/{lang}/{site}.xml"`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guides/en/example.com.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guides/en/example.com.xml')
)
expect(content('tests/__data__/output/guides/fr/example.com.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guides/fr/example.com.xml')
)
})
it('can grab epg then language filter enabled', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example.com/example.com.channels.xml --output=tests/__data__/output/guides/{lang}/{site}.xml --lang=fr`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guides/fr/example.com.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guides/fr/example.com.xml')
)
})
it('can grab epg then using a multi-language filter', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/example.com/example.com.channels.xml --output=tests/__data__/output/guides/{site}.xml --lang=fr,it`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guides/example.com.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guide_4.xml')
)
})
it('can grab epg using custom channels list', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/custom.channels.xml --output=tests/__data__/output/guide.xml`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guide.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guide_3.xml')
content('tests/__data__/expected/epg_grab/base.guide.xml')
)
})
@@ -119,29 +47,112 @@ describe('epg:grab', () => {
expect(errorThrown).toBe(true)
})
it('can grab epg with wildcard as output', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels="tests/__data__/input/epg_grab/sites/example.com/example.com.channels.xml" --output="tests/__data__/output/guides/{lang}/{site}.xml" --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guides/en/example.com.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guides/en/example.com.xml')
)
expect(content('tests/__data__/output/guides/fr/example.com.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guides/fr/example.com.xml')
)
})
it('can grab epg then language filter enabled', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/example.com/example.com.channels.xml --output=tests/__data__/output/guides/{lang}/{site}.xml --lang=fr --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guides/fr/example.com.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guides/fr/example.com.xml')
)
})
it('can grab epg then using a multi-language filter', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/example.com/example.com.channels.xml --output=tests/__data__/output/guides/{site}.xml --lang=fr,it --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guides/example.com.xml')).toEqual(
content('tests/__data__/expected/epg_grab/lang.guide.xml')
)
})
it('can grab epg via https proxy', () => {
const cmd = `${ENV_VAR} npm run grab --- --site=example.com --proxy=https://bob:123456@proxy.com:1234 --output="${path.resolve(
'tests/__data__/output/guide.xml'
)}"`
)}" --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guide.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guide_2.xml')
content('tests/__data__/expected/epg_grab/proxy.guide.xml')
)
})
it('can grab epg via socks5 proxy', () => {
const cmd = `${ENV_VAR} npm run grab --- --site=example.com --proxy=socks5://bob:123456@proxy.com:1234 --output="${path.resolve(
'tests/__data__/output/guide.xml'
)}"`
)}" --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guide.xml')).toEqual(
content('tests/__data__/expected/epg_grab/guide_2.xml')
content('tests/__data__/expected/epg_grab/proxy.guide.xml')
)
})
it('can grab epg with curl option', () => {
const cmd = `${ENV_VAR} npm run grab --- --site=example.com --curl --output="${path.resolve(
'tests/__data__/output/guide.xml'
)}" --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(stdout).toContain('curl https://example.com')
})
it('can grab epg with multiple channels.xml files', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/**/*.channels.xml --output=tests/__data__/output/guide.xml --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guide.xml')).toEqual(
content('tests/__data__/expected/epg_grab/template.guide.xml')
)
})
it('can grab epg using custom channels list', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/custom.channels.xml --output=tests/__data__/output/guide.xml --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guide.xml')).toEqual(
content('tests/__data__/expected/epg_grab/custom_channels.guide.xml')
)
})
it('can grab epg with gzip option enabled', () => {
const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/sites/**/*.channels.xml --output="${path.resolve(
'tests/__data__/output/guide.xml'
)}" --gzip --timeout=100`
const stdout = execSync(cmd, { encoding: 'utf8' })
if (process.env.DEBUG === 'true') console.log(cmd, stdout)
expect(content('tests/__data__/output/guide.xml')).toEqual(
content('tests/__data__/expected/epg_grab/template.guide.xml')
)
const zip = new Zip()
const expected = zip.decompress(fs.readFileSync('tests/__data__/output/guide.xml.gz'))
const result = zip.decompress(
fs.readFileSync('tests/__data__/expected/epg_grab/template.guide.xml.gz')
)
expect(expected).toEqual(result)
})
})
function content(filepath: string) {