Normalize line endings per .gitattributes
This commit is contained in:
PopeyeTheSai10r committed
1 parent
c5a88bcc6b
commit
4007505adc
1356 files changed
+782784
-782784
No files matched your search
@@ -1,7 +1,7 @@
|
||||
import { downloadData } from '../../api'
|
||||
|
||||
async function main() {
|
||||
await downloadData()
|
||||
}
|
||||
|
||||
main()
|
||||
import { downloadData } from '../../api'
|
||||
|
||||
async function main() {
|
||||
await downloadData()
|
||||
}
|
||||
|
||||
main()
|
||||
+184
-184
@@ -1,184 +1,184 @@
|
||||
import { loadData, data, searchChannels } from '../../api'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { Collection, Logger } from '@freearhey/core'
|
||||
import { select, input } from '@inquirer/prompts'
|
||||
import { generateChannelsXML } from '../../core'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Channel } from '../../models'
|
||||
import * as sdk from '@iptv-org/sdk'
|
||||
import { Command } from 'commander'
|
||||
|
||||
interface ChoiceValue {
|
||||
type: string
|
||||
value?: sdk.Models.Feed | sdk.Models.Channel
|
||||
}
|
||||
interface Choice {
|
||||
name: string
|
||||
short?: string
|
||||
value: ChoiceValue
|
||||
default?: boolean
|
||||
}
|
||||
|
||||
const program = new Command()
|
||||
|
||||
program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(process.argv)
|
||||
|
||||
const filepath = program.args[0]
|
||||
const logger = new Logger()
|
||||
const storage = new Storage()
|
||||
let channelsFromXML = new Collection<Channel>()
|
||||
|
||||
main(filepath)
|
||||
process.on('SIGINT', () => {
|
||||
save(filepath, channelsFromXML)
|
||||
if (process.platform === 'win32') process.kill(0)
|
||||
})
|
||||
|
||||
export default async function main(filepath: string) {
|
||||
if (!(await storage.exists(filepath))) {
|
||||
throw new Error(`File "${filepath}" does not exists`)
|
||||
}
|
||||
|
||||
logger.info('loading data from api...')
|
||||
await loadData()
|
||||
|
||||
logger.info('loading channels...')
|
||||
const xml = await storage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
const channelsFromXMLWithoutId = channelsFromXML.filter((channel: Channel) => !channel.xmltv_id)
|
||||
|
||||
logger.info(
|
||||
`found ${channelsFromXML.count()} channels (including ${channelsFromXMLWithoutId.count()} without ID)`
|
||||
)
|
||||
|
||||
logger.info('starting...')
|
||||
console.log()
|
||||
|
||||
for (const channel of channelsFromXMLWithoutId.all()) {
|
||||
try {
|
||||
channel.xmltv_id = await selectChannel(channel)
|
||||
} catch {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
channelsFromXMLWithoutId.forEach((channel: epgGrabber.Channel) => {
|
||||
if (channel.xmltv_id === '-') {
|
||||
channel.xmltv_id = ''
|
||||
}
|
||||
})
|
||||
save(filepath, channelsFromXML)
|
||||
}
|
||||
|
||||
async function selectChannel(channel: epgGrabber.Channel): Promise<string> {
|
||||
const similarChannels = searchChannels(channel.name)
|
||||
const choices = getChoicesForChannel(similarChannels).all()
|
||||
|
||||
const selected: ChoiceValue = await select({
|
||||
message: `Select channel ID for "${channel.name}" (${channel.site_id}):`,
|
||||
choices,
|
||||
pageSize: 10
|
||||
})
|
||||
|
||||
switch (selected.type) {
|
||||
case 'skip':
|
||||
return '-'
|
||||
case 'type': {
|
||||
const typedChannelId = await input({ message: ' Channel ID:' })
|
||||
if (!typedChannelId) return ''
|
||||
const selectedFeedId = await selectFeed(typedChannelId)
|
||||
if (selectedFeedId === '-') return typedChannelId
|
||||
return [typedChannelId, selectedFeedId].join('@')
|
||||
}
|
||||
case 'channel': {
|
||||
const selectedChannel = selected.value
|
||||
if (!selectedChannel) return ''
|
||||
const selectedFeedId = await selectFeed(selectedChannel.id || '')
|
||||
if (selectedFeedId === '-') return selectedChannel.id || ''
|
||||
return [selectedChannel.id, selectedFeedId].join('@')
|
||||
}
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
async function selectFeed(channelId: string): Promise<string> {
|
||||
const channelFeeds = new Collection(data.feedsGroupedByChannelId.get(channelId))
|
||||
const choices = getChoicesForFeed(channelFeeds).all()
|
||||
|
||||
const selected: ChoiceValue = await select({
|
||||
message: `Select feed ID for "${channelId}":`,
|
||||
choices,
|
||||
pageSize: 10
|
||||
})
|
||||
|
||||
switch (selected.type) {
|
||||
case 'skip':
|
||||
return '-'
|
||||
case 'type':
|
||||
return await input({ message: ' Feed ID:', default: 'SD' })
|
||||
case 'feed':
|
||||
const selectedFeed = selected.value
|
||||
if (!selectedFeed) return ''
|
||||
return selectedFeed.id || ''
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
function getChoicesForChannel(channels: Collection<sdk.Models.Channel>): Collection<Choice> {
|
||||
const choices = new Collection<Choice>()
|
||||
|
||||
channels.forEach((channel: sdk.Models.Channel) => {
|
||||
const names = new Collection([channel.name, ...channel.alt_names]).uniq().join(', ')
|
||||
|
||||
choices.add({
|
||||
value: {
|
||||
type: 'channel',
|
||||
value: channel
|
||||
},
|
||||
name: `${channel.id} (${names})`,
|
||||
short: `${channel.id}`
|
||||
})
|
||||
})
|
||||
|
||||
choices.add({ name: 'Type...', value: { type: 'type' } })
|
||||
choices.add({ name: 'Skip', value: { type: 'skip' } })
|
||||
|
||||
return choices
|
||||
}
|
||||
|
||||
function getChoicesForFeed(feeds: Collection<sdk.Models.Feed>): Collection<Choice> {
|
||||
const choices = new Collection<Choice>()
|
||||
|
||||
feeds.forEach((feed: sdk.Models.Feed) => {
|
||||
let name = `${feed.id} (${feed.name})`
|
||||
if (feed.is_main) name += ' [main]'
|
||||
|
||||
choices.add({
|
||||
value: {
|
||||
type: 'feed',
|
||||
value: feed
|
||||
},
|
||||
default: feed.is_main,
|
||||
name,
|
||||
short: feed.id
|
||||
})
|
||||
})
|
||||
|
||||
choices.add({ name: 'Type...', value: { type: 'type' } })
|
||||
choices.add({ name: 'Skip', value: { type: 'skip' } })
|
||||
|
||||
return choices
|
||||
}
|
||||
|
||||
function save(filepath: string, channelsFromXML: Collection<Channel>) {
|
||||
if (!storage.existsSync(filepath)) return
|
||||
const xml = generateChannelsXML(channelsFromXML)
|
||||
storage.saveSync(filepath, xml)
|
||||
console.log()
|
||||
logger.info(`File '${filepath}' successfully saved`)
|
||||
}
|
||||
import { loadData, data, searchChannels } from '../../api'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { Collection, Logger } from '@freearhey/core'
|
||||
import { select, input } from '@inquirer/prompts'
|
||||
import { generateChannelsXML } from '../../core'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Channel } from '../../models'
|
||||
import * as sdk from '@iptv-org/sdk'
|
||||
import { Command } from 'commander'
|
||||
|
||||
interface ChoiceValue {
|
||||
type: string
|
||||
value?: sdk.Models.Feed | sdk.Models.Channel
|
||||
}
|
||||
interface Choice {
|
||||
name: string
|
||||
short?: string
|
||||
value: ChoiceValue
|
||||
default?: boolean
|
||||
}
|
||||
|
||||
const program = new Command()
|
||||
|
||||
program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(process.argv)
|
||||
|
||||
const filepath = program.args[0]
|
||||
const logger = new Logger()
|
||||
const storage = new Storage()
|
||||
let channelsFromXML = new Collection<Channel>()
|
||||
|
||||
main(filepath)
|
||||
process.on('SIGINT', () => {
|
||||
save(filepath, channelsFromXML)
|
||||
if (process.platform === 'win32') process.kill(0)
|
||||
})
|
||||
|
||||
export default async function main(filepath: string) {
|
||||
if (!(await storage.exists(filepath))) {
|
||||
throw new Error(`File "${filepath}" does not exists`)
|
||||
}
|
||||
|
||||
logger.info('loading data from api...')
|
||||
await loadData()
|
||||
|
||||
logger.info('loading channels...')
|
||||
const xml = await storage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
const channelsFromXMLWithoutId = channelsFromXML.filter((channel: Channel) => !channel.xmltv_id)
|
||||
|
||||
logger.info(
|
||||
`found ${channelsFromXML.count()} channels (including ${channelsFromXMLWithoutId.count()} without ID)`
|
||||
)
|
||||
|
||||
logger.info('starting...')
|
||||
console.log()
|
||||
|
||||
for (const channel of channelsFromXMLWithoutId.all()) {
|
||||
try {
|
||||
channel.xmltv_id = await selectChannel(channel)
|
||||
} catch {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
channelsFromXMLWithoutId.forEach((channel: epgGrabber.Channel) => {
|
||||
if (channel.xmltv_id === '-') {
|
||||
channel.xmltv_id = ''
|
||||
}
|
||||
})
|
||||
save(filepath, channelsFromXML)
|
||||
}
|
||||
|
||||
async function selectChannel(channel: epgGrabber.Channel): Promise<string> {
|
||||
const similarChannels = searchChannels(channel.name)
|
||||
const choices = getChoicesForChannel(similarChannels).all()
|
||||
|
||||
const selected: ChoiceValue = await select({
|
||||
message: `Select channel ID for "${channel.name}" (${channel.site_id}):`,
|
||||
choices,
|
||||
pageSize: 10
|
||||
})
|
||||
|
||||
switch (selected.type) {
|
||||
case 'skip':
|
||||
return '-'
|
||||
case 'type': {
|
||||
const typedChannelId = await input({ message: ' Channel ID:' })
|
||||
if (!typedChannelId) return ''
|
||||
const selectedFeedId = await selectFeed(typedChannelId)
|
||||
if (selectedFeedId === '-') return typedChannelId
|
||||
return [typedChannelId, selectedFeedId].join('@')
|
||||
}
|
||||
case 'channel': {
|
||||
const selectedChannel = selected.value
|
||||
if (!selectedChannel) return ''
|
||||
const selectedFeedId = await selectFeed(selectedChannel.id || '')
|
||||
if (selectedFeedId === '-') return selectedChannel.id || ''
|
||||
return [selectedChannel.id, selectedFeedId].join('@')
|
||||
}
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
async function selectFeed(channelId: string): Promise<string> {
|
||||
const channelFeeds = new Collection(data.feedsGroupedByChannelId.get(channelId))
|
||||
const choices = getChoicesForFeed(channelFeeds).all()
|
||||
|
||||
const selected: ChoiceValue = await select({
|
||||
message: `Select feed ID for "${channelId}":`,
|
||||
choices,
|
||||
pageSize: 10
|
||||
})
|
||||
|
||||
switch (selected.type) {
|
||||
case 'skip':
|
||||
return '-'
|
||||
case 'type':
|
||||
return await input({ message: ' Feed ID:', default: 'SD' })
|
||||
case 'feed':
|
||||
const selectedFeed = selected.value
|
||||
if (!selectedFeed) return ''
|
||||
return selectedFeed.id || ''
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
function getChoicesForChannel(channels: Collection<sdk.Models.Channel>): Collection<Choice> {
|
||||
const choices = new Collection<Choice>()
|
||||
|
||||
channels.forEach((channel: sdk.Models.Channel) => {
|
||||
const names = new Collection([channel.name, ...channel.alt_names]).uniq().join(', ')
|
||||
|
||||
choices.add({
|
||||
value: {
|
||||
type: 'channel',
|
||||
value: channel
|
||||
},
|
||||
name: `${channel.id} (${names})`,
|
||||
short: `${channel.id}`
|
||||
})
|
||||
})
|
||||
|
||||
choices.add({ name: 'Type...', value: { type: 'type' } })
|
||||
choices.add({ name: 'Skip', value: { type: 'skip' } })
|
||||
|
||||
return choices
|
||||
}
|
||||
|
||||
function getChoicesForFeed(feeds: Collection<sdk.Models.Feed>): Collection<Choice> {
|
||||
const choices = new Collection<Choice>()
|
||||
|
||||
feeds.forEach((feed: sdk.Models.Feed) => {
|
||||
let name = `${feed.id} (${feed.name})`
|
||||
if (feed.is_main) name += ' [main]'
|
||||
|
||||
choices.add({
|
||||
value: {
|
||||
type: 'feed',
|
||||
value: feed
|
||||
},
|
||||
default: feed.is_main,
|
||||
name,
|
||||
short: feed.id
|
||||
})
|
||||
})
|
||||
|
||||
choices.add({ name: 'Type...', value: { type: 'type' } })
|
||||
choices.add({ name: 'Skip', value: { type: 'skip' } })
|
||||
|
||||
return choices
|
||||
}
|
||||
|
||||
function save(filepath: string, channelsFromXML: Collection<Channel>) {
|
||||
if (!storage.existsSync(filepath)) return
|
||||
const xml = generateChannelsXML(channelsFromXML)
|
||||
storage.saveSync(filepath, xml)
|
||||
console.log()
|
||||
logger.info(`File '${filepath}' successfully saved`)
|
||||
}
|
||||
@@ -1,60 +1,60 @@
|
||||
import { Collection, Logger } from '@freearhey/core'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { generateChannelsXML } from '../../core'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { SITES_DIR } from '../../constants'
|
||||
import { data, loadData } from '../../api'
|
||||
import { Channel } from '../../models'
|
||||
import { program } from 'commander'
|
||||
|
||||
program.argument('[filepath...]', 'Path to file to format').parse(process.argv)
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info('loading data from api...')
|
||||
await loadData()
|
||||
|
||||
logger.info('loading *.channels.xml files...')
|
||||
const storage = new Storage()
|
||||
const files = program.args.length
|
||||
? program.args
|
||||
: await storage.list(`${SITES_DIR}/**/*.channels.xml`)
|
||||
|
||||
logger.info(`found ${files.length} file(s)`)
|
||||
|
||||
logger.info('formating...')
|
||||
for (const filepath of files) {
|
||||
if (!storage.existsSync(filepath)) continue
|
||||
|
||||
const xml = await storage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
const channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
channelsFromXML.forEach((channel: Channel) => {
|
||||
if (!channel.xmltv_id) return
|
||||
if (data.feedsKeyByStreamId.get(channel.xmltv_id)) return
|
||||
|
||||
const channelData = data.channelsKeyById.get(channel.xmltv_id)
|
||||
if (channelData) {
|
||||
const mainFeed = channelData.getMainFeed()
|
||||
if (mainFeed) {
|
||||
channel.xmltv_id = mainFeed.getStreamId()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
channel.xmltv_id = ''
|
||||
})
|
||||
|
||||
channelsFromXML.sortBy((channel: Channel) => channel.site_id)
|
||||
|
||||
const output = generateChannelsXML(channelsFromXML)
|
||||
|
||||
await storage.save(filepath, output)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
import { Collection, Logger } from '@freearhey/core'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { generateChannelsXML } from '../../core'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { SITES_DIR } from '../../constants'
|
||||
import { data, loadData } from '../../api'
|
||||
import { Channel } from '../../models'
|
||||
import { program } from 'commander'
|
||||
|
||||
program.argument('[filepath...]', 'Path to file to format').parse(process.argv)
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info('loading data from api...')
|
||||
await loadData()
|
||||
|
||||
logger.info('loading *.channels.xml files...')
|
||||
const storage = new Storage()
|
||||
const files = program.args.length
|
||||
? program.args
|
||||
: await storage.list(`${SITES_DIR}/**/*.channels.xml`)
|
||||
|
||||
logger.info(`found ${files.length} file(s)`)
|
||||
|
||||
logger.info('formating...')
|
||||
for (const filepath of files) {
|
||||
if (!storage.existsSync(filepath)) continue
|
||||
|
||||
const xml = await storage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
const channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
channelsFromXML.forEach((channel: Channel) => {
|
||||
if (!channel.xmltv_id) return
|
||||
if (data.feedsKeyByStreamId.get(channel.xmltv_id)) return
|
||||
|
||||
const channelData = data.channelsKeyById.get(channel.xmltv_id)
|
||||
if (channelData) {
|
||||
const mainFeed = channelData.getMainFeed()
|
||||
if (mainFeed) {
|
||||
channel.xmltv_id = mainFeed.getStreamId()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
channel.xmltv_id = ''
|
||||
})
|
||||
|
||||
channelsFromXML.sortBy((channel: Channel) => channel.site_id)
|
||||
|
||||
const output = generateChannelsXML(channelsFromXML)
|
||||
|
||||
await storage.save(filepath, output)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
+109
-109
@@ -1,109 +1,109 @@
|
||||
import { XmlDocument, XsdValidator, XmlValidateError, ErrorDetail } from 'libxml2-wasm'
|
||||
import { Storage, File } from '@freearhey/storage-js'
|
||||
import { program } from 'commander'
|
||||
import chalk from 'chalk'
|
||||
|
||||
const xsd = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
|
||||
<xs:element name="channels">
|
||||
<xs:complexType>
|
||||
<xs:sequence>
|
||||
<xs:element minOccurs="0" maxOccurs="unbounded" ref="channel"/>
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:element name="channel">
|
||||
<xs:complexType mixed="true">
|
||||
<xs:attribute use="required" ref="site"/>
|
||||
<xs:attribute use="required" ref="lang"/>
|
||||
<xs:attribute use="required" ref="site_id"/>
|
||||
<xs:attribute name="xmltv_id" use="required" type="xs:string"/>
|
||||
<xs:attribute name="logo" type="xs:string"/>
|
||||
<xs:attribute name="lcn" type="xs:string"/>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:attribute name="site">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="site_id">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="lang">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
</xs:schema>`
|
||||
|
||||
program.argument('[filepath...]', 'Path to *.channels.xml files to check').parse(process.argv)
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage()
|
||||
|
||||
let errors: ErrorDetail[] = []
|
||||
|
||||
const files = program.args.length ? program.args : await storage.list('sites/**/*.channels.xml')
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'xml') continue
|
||||
|
||||
const xml = await storage.load(filepath)
|
||||
|
||||
let localErrors: ErrorDetail[] = []
|
||||
|
||||
try {
|
||||
const schema = XmlDocument.fromString(xsd)
|
||||
const validator = XsdValidator.fromDoc(schema)
|
||||
const doc = XmlDocument.fromString(xml)
|
||||
|
||||
validator.validate(doc)
|
||||
|
||||
schema.dispose()
|
||||
validator.dispose()
|
||||
doc.dispose()
|
||||
} catch (_error) {
|
||||
const error = _error as XmlValidateError
|
||||
|
||||
localErrors = localErrors.concat(error.details)
|
||||
}
|
||||
|
||||
xml.split('\n').forEach((line: string, lineIndex: number) => {
|
||||
const found = line.match(/='/)
|
||||
if (found) {
|
||||
const colIndex = found.index || 0
|
||||
localErrors.push({
|
||||
line: lineIndex + 1,
|
||||
col: colIndex + 1,
|
||||
message: 'Single quotes cannot be used in attributes'
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if (localErrors.length) {
|
||||
console.log(`\n${chalk.underline(filepath)}`)
|
||||
localErrors.forEach((error: ErrorDetail) => {
|
||||
const position = `${error.line}:${error.col}`
|
||||
console.log(` ${chalk.gray(position.padEnd(4, ' '))} ${error.message.trim()}`)
|
||||
})
|
||||
|
||||
errors = errors.concat(localErrors)
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
console.log(chalk.red(`\n${errors.length} error(s)`))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
import { XmlDocument, XsdValidator, XmlValidateError, ErrorDetail } from 'libxml2-wasm'
|
||||
import { Storage, File } from '@freearhey/storage-js'
|
||||
import { program } from 'commander'
|
||||
import chalk from 'chalk'
|
||||
|
||||
const xsd = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
|
||||
<xs:element name="channels">
|
||||
<xs:complexType>
|
||||
<xs:sequence>
|
||||
<xs:element minOccurs="0" maxOccurs="unbounded" ref="channel"/>
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:element name="channel">
|
||||
<xs:complexType mixed="true">
|
||||
<xs:attribute use="required" ref="site"/>
|
||||
<xs:attribute use="required" ref="lang"/>
|
||||
<xs:attribute use="required" ref="site_id"/>
|
||||
<xs:attribute name="xmltv_id" use="required" type="xs:string"/>
|
||||
<xs:attribute name="logo" type="xs:string"/>
|
||||
<xs:attribute name="lcn" type="xs:string"/>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:attribute name="site">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="site_id">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="lang">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
</xs:schema>`
|
||||
|
||||
program.argument('[filepath...]', 'Path to *.channels.xml files to check').parse(process.argv)
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage()
|
||||
|
||||
let errors: ErrorDetail[] = []
|
||||
|
||||
const files = program.args.length ? program.args : await storage.list('sites/**/*.channels.xml')
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'xml') continue
|
||||
|
||||
const xml = await storage.load(filepath)
|
||||
|
||||
let localErrors: ErrorDetail[] = []
|
||||
|
||||
try {
|
||||
const schema = XmlDocument.fromString(xsd)
|
||||
const validator = XsdValidator.fromDoc(schema)
|
||||
const doc = XmlDocument.fromString(xml)
|
||||
|
||||
validator.validate(doc)
|
||||
|
||||
schema.dispose()
|
||||
validator.dispose()
|
||||
doc.dispose()
|
||||
} catch (_error) {
|
||||
const error = _error as XmlValidateError
|
||||
|
||||
localErrors = localErrors.concat(error.details)
|
||||
}
|
||||
|
||||
xml.split('\n').forEach((line: string, lineIndex: number) => {
|
||||
const found = line.match(/='/)
|
||||
if (found) {
|
||||
const colIndex = found.index || 0
|
||||
localErrors.push({
|
||||
line: lineIndex + 1,
|
||||
col: colIndex + 1,
|
||||
message: 'Single quotes cannot be used in attributes'
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if (localErrors.length) {
|
||||
console.log(`\n${chalk.underline(filepath)}`)
|
||||
localErrors.forEach((error: ErrorDetail) => {
|
||||
const position = `${error.line}:${error.col}`
|
||||
console.log(` ${chalk.gray(position.padEnd(4, ' '))} ${error.message.trim()}`)
|
||||
})
|
||||
|
||||
errors = errors.concat(localErrors)
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
console.log(chalk.red(`\n${errors.length} error(s)`))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
+119
-119
@@ -1,119 +1,119 @@
|
||||
import { Storage, File } from '@freearhey/storage-js'
|
||||
import { Collection, Logger } from '@freearhey/core'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { generateChannelsXML } from '../../core'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { Channel } from '../../models'
|
||||
import { Command } from 'commander'
|
||||
|
||||
interface SiteConfigChannelData {
|
||||
xmltv_id: string
|
||||
name: string
|
||||
site_id: string
|
||||
lang?: string
|
||||
logo?: string
|
||||
url?: string
|
||||
lcn?: string
|
||||
}
|
||||
|
||||
const program = new Command()
|
||||
program
|
||||
.requiredOption('-c, --config <config>', 'Config file')
|
||||
.option('-s, --set [args...]', 'Set custom arguments')
|
||||
.option('-o, --output <output>', 'Output file')
|
||||
.parse(process.argv)
|
||||
|
||||
interface ParseOptions {
|
||||
config: string
|
||||
set?: string
|
||||
output?: string
|
||||
clean?: boolean
|
||||
}
|
||||
|
||||
const options: ParseOptions = program.opts()
|
||||
|
||||
async function main() {
|
||||
function isPromise(promise: object[] | Promise<object[]>) {
|
||||
return (
|
||||
!!promise &&
|
||||
typeof promise === 'object' &&
|
||||
typeof (promise as Promise<object[]>).then === 'function'
|
||||
)
|
||||
}
|
||||
|
||||
const storage = new Storage()
|
||||
const logger = new Logger()
|
||||
const file = new File(options.config)
|
||||
const dir = file.dirname()
|
||||
const config = (await import(pathToFileURL(options.config).toString())).default
|
||||
const outputFilepath = options.output || `${dir}/${config.site}.channels.xml`
|
||||
|
||||
const args: Record<string, string> = {}
|
||||
|
||||
if (Array.isArray(options.set)) {
|
||||
options.set.forEach((arg: string) => {
|
||||
const [key, value] = arg.split(':')
|
||||
args[key] = value
|
||||
})
|
||||
}
|
||||
|
||||
let channelsFromXML = new Collection<Channel>()
|
||||
if (await storage.exists(outputFilepath)) {
|
||||
const xml = await storage.load(outputFilepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
}
|
||||
|
||||
let configChannels = config.channels(args)
|
||||
if (isPromise(configChannels)) {
|
||||
configChannels = await configChannels
|
||||
}
|
||||
|
||||
const channelsFromConfig = new Collection<SiteConfigChannelData>(configChannels).map(
|
||||
(data: SiteConfigChannelData) => {
|
||||
return new Channel({
|
||||
xmltv_id: data.xmltv_id,
|
||||
name: data.name,
|
||||
site_id: data.site_id,
|
||||
lang: data.lang || null,
|
||||
logo: data.logo || null,
|
||||
url: data.url || null,
|
||||
lcn: data.lcn || null,
|
||||
site: config.site,
|
||||
index: -1
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
const newChannelList = new Collection<Channel>()
|
||||
channelsFromConfig.forEach((channel: Channel) => {
|
||||
if (!channel.site_id) return
|
||||
|
||||
const found: Channel | undefined = channelsFromXML.find(
|
||||
(_channel: Channel) => _channel.site_id == channel.site_id
|
||||
)
|
||||
|
||||
if (found) {
|
||||
channel.xmltv_id = found.xmltv_id
|
||||
channel.lang = found.lang
|
||||
}
|
||||
|
||||
newChannelList.add(channel)
|
||||
})
|
||||
|
||||
newChannelList.sortBy([
|
||||
(channel: Channel) => channel.lang || '_',
|
||||
(channel: Channel) => (channel.xmltv_id ? channel.xmltv_id.toLowerCase() : '0'),
|
||||
(channel: Channel) => channel.site_id
|
||||
])
|
||||
|
||||
const xml = generateChannelsXML(newChannelList)
|
||||
|
||||
await storage.save(outputFilepath, xml)
|
||||
|
||||
logger.info(`File '${outputFilepath}' successfully saved`)
|
||||
}
|
||||
|
||||
main()
|
||||
import { Storage, File } from '@freearhey/storage-js'
|
||||
import { Collection, Logger } from '@freearhey/core'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { generateChannelsXML } from '../../core'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { Channel } from '../../models'
|
||||
import { Command } from 'commander'
|
||||
|
||||
interface SiteConfigChannelData {
|
||||
xmltv_id: string
|
||||
name: string
|
||||
site_id: string
|
||||
lang?: string
|
||||
logo?: string
|
||||
url?: string
|
||||
lcn?: string
|
||||
}
|
||||
|
||||
const program = new Command()
|
||||
program
|
||||
.requiredOption('-c, --config <config>', 'Config file')
|
||||
.option('-s, --set [args...]', 'Set custom arguments')
|
||||
.option('-o, --output <output>', 'Output file')
|
||||
.parse(process.argv)
|
||||
|
||||
interface ParseOptions {
|
||||
config: string
|
||||
set?: string
|
||||
output?: string
|
||||
clean?: boolean
|
||||
}
|
||||
|
||||
const options: ParseOptions = program.opts()
|
||||
|
||||
async function main() {
|
||||
function isPromise(promise: object[] | Promise<object[]>) {
|
||||
return (
|
||||
!!promise &&
|
||||
typeof promise === 'object' &&
|
||||
typeof (promise as Promise<object[]>).then === 'function'
|
||||
)
|
||||
}
|
||||
|
||||
const storage = new Storage()
|
||||
const logger = new Logger()
|
||||
const file = new File(options.config)
|
||||
const dir = file.dirname()
|
||||
const config = (await import(pathToFileURL(options.config).toString())).default
|
||||
const outputFilepath = options.output || `${dir}/${config.site}.channels.xml`
|
||||
|
||||
const args: Record<string, string> = {}
|
||||
|
||||
if (Array.isArray(options.set)) {
|
||||
options.set.forEach((arg: string) => {
|
||||
const [key, value] = arg.split(':')
|
||||
args[key] = value
|
||||
})
|
||||
}
|
||||
|
||||
let channelsFromXML = new Collection<Channel>()
|
||||
if (await storage.exists(outputFilepath)) {
|
||||
const xml = await storage.load(outputFilepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
}
|
||||
|
||||
let configChannels = config.channels(args)
|
||||
if (isPromise(configChannels)) {
|
||||
configChannels = await configChannels
|
||||
}
|
||||
|
||||
const channelsFromConfig = new Collection<SiteConfigChannelData>(configChannels).map(
|
||||
(data: SiteConfigChannelData) => {
|
||||
return new Channel({
|
||||
xmltv_id: data.xmltv_id,
|
||||
name: data.name,
|
||||
site_id: data.site_id,
|
||||
lang: data.lang || null,
|
||||
logo: data.logo || null,
|
||||
url: data.url || null,
|
||||
lcn: data.lcn || null,
|
||||
site: config.site,
|
||||
index: -1
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
const newChannelList = new Collection<Channel>()
|
||||
channelsFromConfig.forEach((channel: Channel) => {
|
||||
if (!channel.site_id) return
|
||||
|
||||
const found: Channel | undefined = channelsFromXML.find(
|
||||
(_channel: Channel) => _channel.site_id == channel.site_id
|
||||
)
|
||||
|
||||
if (found) {
|
||||
channel.xmltv_id = found.xmltv_id
|
||||
channel.lang = found.lang
|
||||
}
|
||||
|
||||
newChannelList.add(channel)
|
||||
})
|
||||
|
||||
newChannelList.sortBy([
|
||||
(channel: Channel) => channel.lang || '_',
|
||||
(channel: Channel) => (channel.xmltv_id ? channel.xmltv_id.toLowerCase() : '0'),
|
||||
(channel: Channel) => channel.site_id
|
||||
])
|
||||
|
||||
const xml = generateChannelsXML(newChannelList)
|
||||
|
||||
await storage.save(outputFilepath, xml)
|
||||
|
||||
logger.info(`File '${outputFilepath}' successfully saved`)
|
||||
}
|
||||
|
||||
main()
|
||||
@@ -1,96 +1,96 @@
|
||||
import { Collection, Dictionary } from '@freearhey/core'
|
||||
import { Storage, File } from '@freearhey/storage-js'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { loadData, data } from '../../api'
|
||||
import { Channel } from '../../models'
|
||||
import { program } from 'commander'
|
||||
import chalk from 'chalk'
|
||||
import langs from 'langs'
|
||||
|
||||
program.argument('[filepath...]', 'Path to *.channels.xml files to validate').parse(process.argv)
|
||||
|
||||
interface ValidationError {
|
||||
type: 'duplicate' | 'wrong_channel_id' | 'wrong_feed_id' | 'wrong_lang'
|
||||
name: string
|
||||
lang: string | null
|
||||
xmltv_id: string | null
|
||||
site_id: string | null
|
||||
logo: string | null
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await loadData()
|
||||
const { channelsKeyById, feedsKeyByStreamId } = data
|
||||
|
||||
let totalFiles = 0
|
||||
let totalErrors = 0
|
||||
let totalWarnings = 0
|
||||
|
||||
const storage = new Storage()
|
||||
const files = program.args.length ? program.args : await storage.list('sites/**/*.channels.xml')
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'xml') continue
|
||||
|
||||
const xml = await storage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
const channelList = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
const bufferBySiteId = new Dictionary()
|
||||
const errors: ValidationError[] = []
|
||||
channelList.forEach((channel: Channel) => {
|
||||
const bufferId: string = channel.site_id
|
||||
if (bufferBySiteId.missing(bufferId)) {
|
||||
bufferBySiteId.set(bufferId, true)
|
||||
} else {
|
||||
errors.push({ type: 'duplicate', ...channel.toObject() })
|
||||
totalErrors++
|
||||
}
|
||||
|
||||
if (!langs.where('1', channel.lang ?? '')) {
|
||||
errors.push({ type: 'wrong_lang', ...channel.toObject() })
|
||||
totalErrors++
|
||||
}
|
||||
|
||||
if (!channel.xmltv_id) return
|
||||
const [channelId, feedId] = channel.xmltv_id.split('@')
|
||||
|
||||
const foundChannel = channelsKeyById.get(channelId)
|
||||
if (!foundChannel) {
|
||||
errors.push({ type: 'wrong_channel_id', ...channel.toObject() })
|
||||
totalWarnings++
|
||||
}
|
||||
|
||||
if (feedId) {
|
||||
const foundFeed = feedsKeyByStreamId.get(channel.xmltv_id)
|
||||
if (!foundFeed) {
|
||||
errors.push({ type: 'wrong_feed_id', ...channel.toObject() })
|
||||
totalWarnings++
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if (errors.length) {
|
||||
console.log(chalk.underline(filepath))
|
||||
console.table(errors, ['type', 'lang', 'xmltv_id', 'site_id', 'name'])
|
||||
console.log()
|
||||
totalFiles++
|
||||
}
|
||||
}
|
||||
|
||||
const totalProblems = totalWarnings + totalErrors
|
||||
if (totalProblems > 0) {
|
||||
console.log(
|
||||
chalk.red(
|
||||
`${totalProblems} problems (${totalErrors} errors, ${totalWarnings} warnings) in ${totalFiles} file(s)`
|
||||
)
|
||||
)
|
||||
if (totalErrors > 0) {
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
import { Collection, Dictionary } from '@freearhey/core'
|
||||
import { Storage, File } from '@freearhey/storage-js'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { loadData, data } from '../../api'
|
||||
import { Channel } from '../../models'
|
||||
import { program } from 'commander'
|
||||
import chalk from 'chalk'
|
||||
import langs from 'langs'
|
||||
|
||||
program.argument('[filepath...]', 'Path to *.channels.xml files to validate').parse(process.argv)
|
||||
|
||||
interface ValidationError {
|
||||
type: 'duplicate' | 'wrong_channel_id' | 'wrong_feed_id' | 'wrong_lang'
|
||||
name: string
|
||||
lang: string | null
|
||||
xmltv_id: string | null
|
||||
site_id: string | null
|
||||
logo: string | null
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await loadData()
|
||||
const { channelsKeyById, feedsKeyByStreamId } = data
|
||||
|
||||
let totalFiles = 0
|
||||
let totalErrors = 0
|
||||
let totalWarnings = 0
|
||||
|
||||
const storage = new Storage()
|
||||
const files = program.args.length ? program.args : await storage.list('sites/**/*.channels.xml')
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'xml') continue
|
||||
|
||||
const xml = await storage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
const channelList = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
const bufferBySiteId = new Dictionary()
|
||||
const errors: ValidationError[] = []
|
||||
channelList.forEach((channel: Channel) => {
|
||||
const bufferId: string = channel.site_id
|
||||
if (bufferBySiteId.missing(bufferId)) {
|
||||
bufferBySiteId.set(bufferId, true)
|
||||
} else {
|
||||
errors.push({ type: 'duplicate', ...channel.toObject() })
|
||||
totalErrors++
|
||||
}
|
||||
|
||||
if (!langs.where('1', channel.lang ?? '')) {
|
||||
errors.push({ type: 'wrong_lang', ...channel.toObject() })
|
||||
totalErrors++
|
||||
}
|
||||
|
||||
if (!channel.xmltv_id) return
|
||||
const [channelId, feedId] = channel.xmltv_id.split('@')
|
||||
|
||||
const foundChannel = channelsKeyById.get(channelId)
|
||||
if (!foundChannel) {
|
||||
errors.push({ type: 'wrong_channel_id', ...channel.toObject() })
|
||||
totalWarnings++
|
||||
}
|
||||
|
||||
if (feedId) {
|
||||
const foundFeed = feedsKeyByStreamId.get(channel.xmltv_id)
|
||||
if (!foundFeed) {
|
||||
errors.push({ type: 'wrong_feed_id', ...channel.toObject() })
|
||||
totalWarnings++
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if (errors.length) {
|
||||
console.log(chalk.underline(filepath))
|
||||
console.table(errors, ['type', 'lang', 'xmltv_id', 'site_id', 'name'])
|
||||
console.log()
|
||||
totalFiles++
|
||||
}
|
||||
}
|
||||
|
||||
const totalProblems = totalWarnings + totalErrors
|
||||
if (totalProblems > 0) {
|
||||
console.log(
|
||||
chalk.red(
|
||||
`${totalProblems} problems (${totalErrors} errors, ${totalWarnings} warnings) in ${totalFiles} file(s)`
|
||||
)
|
||||
)
|
||||
if (totalErrors > 0) {
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
+372
-372
@@ -1,372 +1,372 @@
|
||||
import { Logger, Timer, Collection, Template } from '@freearhey/core'
|
||||
import epgGrabber, { EPGGrabber, EPGGrabberMock } from 'epg-grabber'
|
||||
import { CurlBody } from 'curl-generator/dist/bodies/body'
|
||||
import { Channel, Guide, Program } from '../../models'
|
||||
import { SocksProxyAgent } from 'socks-proxy-agent'
|
||||
import defaultConfig from '../../default.config'
|
||||
import pLimit from 'p-limit'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { CurlGenerator } from 'curl-generator'
|
||||
import { QueueItem } from '../../types/queue'
|
||||
import { Option, program } from 'commander'
|
||||
import { SITES_DIR } from '../../constants'
|
||||
import { data, loadData } from '../../api'
|
||||
import dayjs, { Dayjs } from 'dayjs'
|
||||
import merge from 'lodash.merge'
|
||||
import path from 'path'
|
||||
import {
|
||||
parseBooleanOrString,
|
||||
parseBoolean,
|
||||
parseNumber,
|
||||
parseProxy,
|
||||
parseList,
|
||||
loadJs
|
||||
} from '../../core'
|
||||
|
||||
program
|
||||
.addOption(
|
||||
new Option('-s, --sites <names>', 'A comma-separated list of the sites to parse').argParser(
|
||||
parseList
|
||||
)
|
||||
)
|
||||
.addOption(
|
||||
new Option(
|
||||
'-c, --channels <path>',
|
||||
'Path to *.channels.xml file (required if the "--sites" attribute is not specified)'
|
||||
)
|
||||
)
|
||||
.addOption(new Option('-o, --output <path>', 'Path to output file'))
|
||||
.addOption(new Option('-l, --lang <codes>', 'Filter channels by languages (ISO 639-1 codes)'))
|
||||
.addOption(
|
||||
new Option('-t, --timeout <milliseconds>', 'Override the default timeout for each request')
|
||||
.env('TIMEOUT')
|
||||
.argParser(parseNumber)
|
||||
)
|
||||
.addOption(
|
||||
new Option('-d, --delay <milliseconds>', 'Override the default delay between request')
|
||||
.env('DELAY')
|
||||
.argParser(parseNumber)
|
||||
)
|
||||
.addOption(new Option('-x, --proxy <url>', 'Use the specified proxy').env('PROXY'))
|
||||
.addOption(
|
||||
new Option(
|
||||
'--days <days>',
|
||||
'Override the number of days for which the program will be loaded (defaults to the value from the site config)'
|
||||
)
|
||||
.argParser(parseNumber)
|
||||
.env('DAYS')
|
||||
)
|
||||
.addOption(
|
||||
new Option('--maxConnections <number>', 'Limit on the number of concurrent requests')
|
||||
.argParser(parseNumber)
|
||||
.env('MAX_CONNECTIONS')
|
||||
)
|
||||
.addOption(
|
||||
new Option('--gzip [path]', 'Create a compressed version of the guide as well')
|
||||
.argParser(parseBooleanOrString)
|
||||
.env('GZIP')
|
||||
)
|
||||
.addOption(
|
||||
new Option('--json [path]', 'Create a JSON version of the guide as well')
|
||||
.argParser(parseBooleanOrString)
|
||||
.env('JSON')
|
||||
)
|
||||
.addOption(
|
||||
new Option('--curl', 'Display each request as CURL').argParser(parseBoolean).env('CURL')
|
||||
)
|
||||
.addOption(new Option('--debug', 'Enable debug mode').argParser(parseBoolean).env('DEBUG'))
|
||||
.parse()
|
||||
|
||||
interface GrabOptions {
|
||||
sites?: string[]
|
||||
channels?: string
|
||||
output?: string
|
||||
gzip?: boolean | string
|
||||
json?: boolean | string
|
||||
curl?: boolean
|
||||
debug?: boolean
|
||||
maxConnections?: number
|
||||
timeout?: number
|
||||
delay?: number
|
||||
lang?: string
|
||||
days?: number
|
||||
proxy?: string
|
||||
}
|
||||
|
||||
const options: GrabOptions = program.opts()
|
||||
|
||||
async function main() {
|
||||
if (!Array.isArray(options.sites) && typeof options.channels !== 'string')
|
||||
throw new Error('One of the arguments must be presented: `--sites` or `--channels`')
|
||||
|
||||
const LOG_LEVELS = { info: 3, debug: 4 }
|
||||
const logger = new Logger({ level: options.debug ? LOG_LEVELS['debug'] : LOG_LEVELS['info'] })
|
||||
|
||||
logger.info('starting...')
|
||||
const globalConfig: epgGrabber.Types.SiteConfig = {}
|
||||
|
||||
if (typeof options.timeout === 'number')
|
||||
merge(globalConfig, { request: { timeout: options.timeout } })
|
||||
if (options.proxy !== undefined) {
|
||||
const proxy = parseProxy(options.proxy)
|
||||
if (
|
||||
proxy.protocol &&
|
||||
['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol))
|
||||
) {
|
||||
const socksProxyAgent = new SocksProxyAgent(options.proxy)
|
||||
merge(globalConfig, {
|
||||
request: { httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent }
|
||||
})
|
||||
} else {
|
||||
merge(globalConfig, { request: { proxy } })
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof options.output === 'string') globalConfig.output = options.output
|
||||
if (typeof options.days === 'number') globalConfig.days = options.days
|
||||
if (typeof options.delay === 'number') globalConfig.delay = options.delay
|
||||
if (typeof options.maxConnections === 'number')
|
||||
globalConfig.maxConnections = options.maxConnections
|
||||
if (typeof options.curl === 'boolean') globalConfig.curl = options.curl
|
||||
if (typeof options.gzip === 'boolean' || typeof options.gzip === 'string')
|
||||
globalConfig.gzip = options.gzip
|
||||
if (typeof options.json === 'boolean' || typeof options.json === 'string')
|
||||
globalConfig.json = options.json
|
||||
if (typeof options.debug === 'boolean') globalConfig.debug = options.debug
|
||||
|
||||
logger.debug(`config: ${JSON.stringify(globalConfig, getCircularReplacer(), 2)}`)
|
||||
|
||||
const grabber =
|
||||
process.env.NODE_ENV === 'test'
|
||||
? new EPGGrabberMock(globalConfig)
|
||||
: new EPGGrabber(globalConfig)
|
||||
|
||||
grabber.client.instance.interceptors.request.use(
|
||||
request => {
|
||||
logger.debug(`request: ${JSON.stringify(request, getCircularReplacer(), 2)}`)
|
||||
|
||||
const curl = globalConfig.curl || defaultConfig.curl
|
||||
if (curl) {
|
||||
type AllowedMethods =
|
||||
| 'GET'
|
||||
| 'get'
|
||||
| 'POST'
|
||||
| 'post'
|
||||
| 'PUT'
|
||||
| 'put'
|
||||
| 'PATCH'
|
||||
| 'patch'
|
||||
| 'DELETE'
|
||||
| 'delete'
|
||||
|
||||
const url = request.url || ''
|
||||
const method = request.method ? (request.method as AllowedMethods) : 'GET'
|
||||
const headers = request.headers
|
||||
? (request.headers.toJSON() as Record<string, string>)
|
||||
: undefined
|
||||
const body = request.data ? (request.data as CurlBody) : undefined
|
||||
|
||||
const curl = CurlGenerator({ url, method, headers, body })
|
||||
|
||||
console.log(curl)
|
||||
}
|
||||
|
||||
return request
|
||||
},
|
||||
error => Promise.reject(error)
|
||||
)
|
||||
|
||||
logger.info('loading channels...')
|
||||
const storage = new Storage()
|
||||
|
||||
let files: string[] = []
|
||||
if (Array.isArray(options.sites)) {
|
||||
for (const site of options.sites) {
|
||||
let pattern = path.join(SITES_DIR, site, '*.channels.xml')
|
||||
pattern = pattern.replace(/\\/g, '/')
|
||||
const foundFiles = await storage.list(pattern)
|
||||
foundFiles.forEach((filepath: string) => {
|
||||
files.push(filepath)
|
||||
})
|
||||
}
|
||||
} else if (typeof options.channels === 'string') {
|
||||
files = await storage.list(options.channels)
|
||||
}
|
||||
|
||||
let channelsFromXML = new Collection<Channel>()
|
||||
for (const filepath of files) {
|
||||
const xml = await storage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
const _channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
channelsFromXML.concat(_channelsFromXML)
|
||||
}
|
||||
|
||||
if (typeof options.lang === 'string') {
|
||||
channelsFromXML = channelsFromXML.filter((channel: Channel) => {
|
||||
if (!options.lang) return true
|
||||
|
||||
return channel.lang ? options.lang.includes(channel.lang) : false
|
||||
})
|
||||
}
|
||||
|
||||
logger.info(`found ${channelsFromXML.count()} channel(s)`)
|
||||
|
||||
logger.info('loading api data...')
|
||||
await loadData()
|
||||
|
||||
logger.info('creating queue...')
|
||||
const queue = new Collection<QueueItem>()
|
||||
|
||||
let index = 0
|
||||
for (const channel of channelsFromXML.all()) {
|
||||
channel.index = index++
|
||||
if (!channel.site || !channel.site_id || !channel.name) continue
|
||||
|
||||
const config = merge({}, defaultConfig, await loadJs(channel.getConfigPath()))
|
||||
|
||||
if (!channel.xmltv_id) channel.xmltv_id = channel.site_id
|
||||
|
||||
const days = globalConfig.days || config.days
|
||||
const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString())
|
||||
const dates = Array.from({ length: days }, (_, day) => currDate.add(day, 'd'))
|
||||
|
||||
dates.forEach((date: Dayjs) => {
|
||||
queue.add({
|
||||
channel,
|
||||
date,
|
||||
config: { ...config },
|
||||
error: null
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const maxConnections = globalConfig.maxConnections || defaultConfig.maxConnections
|
||||
const limit = pLimit(maxConnections)
|
||||
|
||||
const channels = new Collection<Channel>()
|
||||
const programs = new Collection<Program>()
|
||||
|
||||
let i = 1
|
||||
const total = queue.count()
|
||||
|
||||
logger.info('run:')
|
||||
const timer = new Timer()
|
||||
timer.start()
|
||||
|
||||
const requests = queue.all().map((queueItem: QueueItem) =>
|
||||
limit(async () => {
|
||||
const { channel, config, date } = queueItem
|
||||
|
||||
if (!channel.logo) {
|
||||
if (config.logo) {
|
||||
channel.logo = await grabber.loadLogo(channel, date)
|
||||
} else {
|
||||
channel.logo = getLogoForChannel(channel)
|
||||
}
|
||||
}
|
||||
|
||||
channels.add(channel)
|
||||
|
||||
const channelPrograms = await grabber.grab(
|
||||
channel,
|
||||
date,
|
||||
config,
|
||||
(context: epgGrabber.Types.GrabCallbackContext, error: Error | null) => {
|
||||
logger.info(
|
||||
` [${i}/${total}] ${context.channel.site} (${context.channel.lang}) - ${
|
||||
context.channel.xmltv_id
|
||||
} - ${context.date.format('MMM D, YYYY')} (${context.programs.length} programs)`
|
||||
)
|
||||
if (i < total) i++
|
||||
|
||||
if (error) {
|
||||
logger.info(` ERR: ${error.message}`)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const _programs = new Collection<epgGrabber.Program>(channelPrograms).map<Program>(
|
||||
program => new Program(program.toObject())
|
||||
)
|
||||
|
||||
programs.concat(_programs)
|
||||
})
|
||||
)
|
||||
|
||||
await Promise.all(requests)
|
||||
|
||||
const output = globalConfig.output || defaultConfig.output
|
||||
|
||||
const pathTemplate = new Template(output)
|
||||
|
||||
const channelsGroupedByKey = channels
|
||||
.uniqBy((channel: Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`)
|
||||
.groupBy((channel: Channel) => {
|
||||
return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' })
|
||||
})
|
||||
|
||||
const programsGroupedByKey = programs
|
||||
.sortBy([(program: Program) => program.channel, (program: Program) => program.start])
|
||||
.groupBy((program: Program) => {
|
||||
const lang =
|
||||
program.titles && program.titles.length && program.titles[0].lang
|
||||
? program.titles[0].lang
|
||||
: 'en'
|
||||
|
||||
return pathTemplate.format({ lang, site: program.site || '' })
|
||||
})
|
||||
|
||||
const gzip = globalConfig.gzip || defaultConfig.gzip
|
||||
const json = globalConfig.json || defaultConfig.json
|
||||
|
||||
for (const groupKey of channelsGroupedByKey.keys()) {
|
||||
const groupChannels = new Collection(channelsGroupedByKey.get(groupKey))
|
||||
const groupPrograms = new Collection(programsGroupedByKey.get(groupKey))
|
||||
const guide = new Guide({
|
||||
filepath: groupKey,
|
||||
gzip,
|
||||
json,
|
||||
channels: groupChannels,
|
||||
programs: groupPrograms
|
||||
})
|
||||
|
||||
await guide.save({ logger })
|
||||
}
|
||||
|
||||
logger.success(` done in ${timer.format('HH[h] mm[m] ss[s]')}`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function getLogoForChannel(channel: Channel): string | null {
|
||||
const feedData = data.feedsKeyByStreamId.get(channel.xmltv_id)
|
||||
if (feedData) {
|
||||
const firstLogo = feedData.getLogos().first()
|
||||
if (firstLogo) return firstLogo.url
|
||||
}
|
||||
|
||||
const [channelId] = channel.xmltv_id.split('@')
|
||||
const channelData = data.channelsKeyById.get(channelId)
|
||||
if (channelData) {
|
||||
const firstLogo = channelData.getLogos().first()
|
||||
if (firstLogo) return firstLogo.url
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function getCircularReplacer() {
|
||||
const seen = new WeakSet()
|
||||
return (key: string, value: any) => {
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
if (seen.has(value)) {
|
||||
return '[Circular]'
|
||||
}
|
||||
seen.add(value)
|
||||
}
|
||||
return value
|
||||
}
|
||||
}
|
||||
import { Logger, Timer, Collection, Template } from '@freearhey/core'
|
||||
import epgGrabber, { EPGGrabber, EPGGrabberMock } from 'epg-grabber'
|
||||
import { CurlBody } from 'curl-generator/dist/bodies/body'
|
||||
import { Channel, Guide, Program } from '../../models'
|
||||
import { SocksProxyAgent } from 'socks-proxy-agent'
|
||||
import defaultConfig from '../../default.config'
|
||||
import pLimit from 'p-limit'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { CurlGenerator } from 'curl-generator'
|
||||
import { QueueItem } from '../../types/queue'
|
||||
import { Option, program } from 'commander'
|
||||
import { SITES_DIR } from '../../constants'
|
||||
import { data, loadData } from '../../api'
|
||||
import dayjs, { Dayjs } from 'dayjs'
|
||||
import merge from 'lodash.merge'
|
||||
import path from 'path'
|
||||
import {
|
||||
parseBooleanOrString,
|
||||
parseBoolean,
|
||||
parseNumber,
|
||||
parseProxy,
|
||||
parseList,
|
||||
loadJs
|
||||
} from '../../core'
|
||||
|
||||
program
|
||||
.addOption(
|
||||
new Option('-s, --sites <names>', 'A comma-separated list of the sites to parse').argParser(
|
||||
parseList
|
||||
)
|
||||
)
|
||||
.addOption(
|
||||
new Option(
|
||||
'-c, --channels <path>',
|
||||
'Path to *.channels.xml file (required if the "--sites" attribute is not specified)'
|
||||
)
|
||||
)
|
||||
.addOption(new Option('-o, --output <path>', 'Path to output file'))
|
||||
.addOption(new Option('-l, --lang <codes>', 'Filter channels by languages (ISO 639-1 codes)'))
|
||||
.addOption(
|
||||
new Option('-t, --timeout <milliseconds>', 'Override the default timeout for each request')
|
||||
.env('TIMEOUT')
|
||||
.argParser(parseNumber)
|
||||
)
|
||||
.addOption(
|
||||
new Option('-d, --delay <milliseconds>', 'Override the default delay between request')
|
||||
.env('DELAY')
|
||||
.argParser(parseNumber)
|
||||
)
|
||||
.addOption(new Option('-x, --proxy <url>', 'Use the specified proxy').env('PROXY'))
|
||||
.addOption(
|
||||
new Option(
|
||||
'--days <days>',
|
||||
'Override the number of days for which the program will be loaded (defaults to the value from the site config)'
|
||||
)
|
||||
.argParser(parseNumber)
|
||||
.env('DAYS')
|
||||
)
|
||||
.addOption(
|
||||
new Option('--maxConnections <number>', 'Limit on the number of concurrent requests')
|
||||
.argParser(parseNumber)
|
||||
.env('MAX_CONNECTIONS')
|
||||
)
|
||||
.addOption(
|
||||
new Option('--gzip [path]', 'Create a compressed version of the guide as well')
|
||||
.argParser(parseBooleanOrString)
|
||||
.env('GZIP')
|
||||
)
|
||||
.addOption(
|
||||
new Option('--json [path]', 'Create a JSON version of the guide as well')
|
||||
.argParser(parseBooleanOrString)
|
||||
.env('JSON')
|
||||
)
|
||||
.addOption(
|
||||
new Option('--curl', 'Display each request as CURL').argParser(parseBoolean).env('CURL')
|
||||
)
|
||||
.addOption(new Option('--debug', 'Enable debug mode').argParser(parseBoolean).env('DEBUG'))
|
||||
.parse()
|
||||
|
||||
interface GrabOptions {
|
||||
sites?: string[]
|
||||
channels?: string
|
||||
output?: string
|
||||
gzip?: boolean | string
|
||||
json?: boolean | string
|
||||
curl?: boolean
|
||||
debug?: boolean
|
||||
maxConnections?: number
|
||||
timeout?: number
|
||||
delay?: number
|
||||
lang?: string
|
||||
days?: number
|
||||
proxy?: string
|
||||
}
|
||||
|
||||
const options: GrabOptions = program.opts()
|
||||
|
||||
async function main() {
|
||||
if (!Array.isArray(options.sites) && typeof options.channels !== 'string')
|
||||
throw new Error('One of the arguments must be presented: `--sites` or `--channels`')
|
||||
|
||||
const LOG_LEVELS = { info: 3, debug: 4 }
|
||||
const logger = new Logger({ level: options.debug ? LOG_LEVELS['debug'] : LOG_LEVELS['info'] })
|
||||
|
||||
logger.info('starting...')
|
||||
const globalConfig: epgGrabber.Types.SiteConfig = {}
|
||||
|
||||
if (typeof options.timeout === 'number')
|
||||
merge(globalConfig, { request: { timeout: options.timeout } })
|
||||
if (options.proxy !== undefined) {
|
||||
const proxy = parseProxy(options.proxy)
|
||||
if (
|
||||
proxy.protocol &&
|
||||
['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol))
|
||||
) {
|
||||
const socksProxyAgent = new SocksProxyAgent(options.proxy)
|
||||
merge(globalConfig, {
|
||||
request: { httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent }
|
||||
})
|
||||
} else {
|
||||
merge(globalConfig, { request: { proxy } })
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof options.output === 'string') globalConfig.output = options.output
|
||||
if (typeof options.days === 'number') globalConfig.days = options.days
|
||||
if (typeof options.delay === 'number') globalConfig.delay = options.delay
|
||||
if (typeof options.maxConnections === 'number')
|
||||
globalConfig.maxConnections = options.maxConnections
|
||||
if (typeof options.curl === 'boolean') globalConfig.curl = options.curl
|
||||
if (typeof options.gzip === 'boolean' || typeof options.gzip === 'string')
|
||||
globalConfig.gzip = options.gzip
|
||||
if (typeof options.json === 'boolean' || typeof options.json === 'string')
|
||||
globalConfig.json = options.json
|
||||
if (typeof options.debug === 'boolean') globalConfig.debug = options.debug
|
||||
|
||||
logger.debug(`config: ${JSON.stringify(globalConfig, getCircularReplacer(), 2)}`)
|
||||
|
||||
const grabber =
|
||||
process.env.NODE_ENV === 'test'
|
||||
? new EPGGrabberMock(globalConfig)
|
||||
: new EPGGrabber(globalConfig)
|
||||
|
||||
grabber.client.instance.interceptors.request.use(
|
||||
request => {
|
||||
logger.debug(`request: ${JSON.stringify(request, getCircularReplacer(), 2)}`)
|
||||
|
||||
const curl = globalConfig.curl || defaultConfig.curl
|
||||
if (curl) {
|
||||
type AllowedMethods =
|
||||
| 'GET'
|
||||
| 'get'
|
||||
| 'POST'
|
||||
| 'post'
|
||||
| 'PUT'
|
||||
| 'put'
|
||||
| 'PATCH'
|
||||
| 'patch'
|
||||
| 'DELETE'
|
||||
| 'delete'
|
||||
|
||||
const url = request.url || ''
|
||||
const method = request.method ? (request.method as AllowedMethods) : 'GET'
|
||||
const headers = request.headers
|
||||
? (request.headers.toJSON() as Record<string, string>)
|
||||
: undefined
|
||||
const body = request.data ? (request.data as CurlBody) : undefined
|
||||
|
||||
const curl = CurlGenerator({ url, method, headers, body })
|
||||
|
||||
console.log(curl)
|
||||
}
|
||||
|
||||
return request
|
||||
},
|
||||
error => Promise.reject(error)
|
||||
)
|
||||
|
||||
logger.info('loading channels...')
|
||||
const storage = new Storage()
|
||||
|
||||
let files: string[] = []
|
||||
if (Array.isArray(options.sites)) {
|
||||
for (const site of options.sites) {
|
||||
let pattern = path.join(SITES_DIR, site, '*.channels.xml')
|
||||
pattern = pattern.replace(/\\/g, '/')
|
||||
const foundFiles = await storage.list(pattern)
|
||||
foundFiles.forEach((filepath: string) => {
|
||||
files.push(filepath)
|
||||
})
|
||||
}
|
||||
} else if (typeof options.channels === 'string') {
|
||||
files = await storage.list(options.channels)
|
||||
}
|
||||
|
||||
let channelsFromXML = new Collection<Channel>()
|
||||
for (const filepath of files) {
|
||||
const xml = await storage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
const _channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
channelsFromXML.concat(_channelsFromXML)
|
||||
}
|
||||
|
||||
if (typeof options.lang === 'string') {
|
||||
channelsFromXML = channelsFromXML.filter((channel: Channel) => {
|
||||
if (!options.lang) return true
|
||||
|
||||
return channel.lang ? options.lang.includes(channel.lang) : false
|
||||
})
|
||||
}
|
||||
|
||||
logger.info(`found ${channelsFromXML.count()} channel(s)`)
|
||||
|
||||
logger.info('loading api data...')
|
||||
await loadData()
|
||||
|
||||
logger.info('creating queue...')
|
||||
const queue = new Collection<QueueItem>()
|
||||
|
||||
let index = 0
|
||||
for (const channel of channelsFromXML.all()) {
|
||||
channel.index = index++
|
||||
if (!channel.site || !channel.site_id || !channel.name) continue
|
||||
|
||||
const config = merge({}, defaultConfig, await loadJs(channel.getConfigPath()))
|
||||
|
||||
if (!channel.xmltv_id) channel.xmltv_id = channel.site_id
|
||||
|
||||
const days = globalConfig.days || config.days
|
||||
const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString())
|
||||
const dates = Array.from({ length: days }, (_, day) => currDate.add(day, 'd'))
|
||||
|
||||
dates.forEach((date: Dayjs) => {
|
||||
queue.add({
|
||||
channel,
|
||||
date,
|
||||
config: { ...config },
|
||||
error: null
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const maxConnections = globalConfig.maxConnections || defaultConfig.maxConnections
|
||||
const limit = pLimit(maxConnections)
|
||||
|
||||
const channels = new Collection<Channel>()
|
||||
const programs = new Collection<Program>()
|
||||
|
||||
let i = 1
|
||||
const total = queue.count()
|
||||
|
||||
logger.info('run:')
|
||||
const timer = new Timer()
|
||||
timer.start()
|
||||
|
||||
const requests = queue.all().map((queueItem: QueueItem) =>
|
||||
limit(async () => {
|
||||
const { channel, config, date } = queueItem
|
||||
|
||||
if (!channel.logo) {
|
||||
if (config.logo) {
|
||||
channel.logo = await grabber.loadLogo(channel, date)
|
||||
} else {
|
||||
channel.logo = getLogoForChannel(channel)
|
||||
}
|
||||
}
|
||||
|
||||
channels.add(channel)
|
||||
|
||||
const channelPrograms = await grabber.grab(
|
||||
channel,
|
||||
date,
|
||||
config,
|
||||
(context: epgGrabber.Types.GrabCallbackContext, error: Error | null) => {
|
||||
logger.info(
|
||||
` [${i}/${total}] ${context.channel.site} (${context.channel.lang}) - ${
|
||||
context.channel.xmltv_id
|
||||
} - ${context.date.format('MMM D, YYYY')} (${context.programs.length} programs)`
|
||||
)
|
||||
if (i < total) i++
|
||||
|
||||
if (error) {
|
||||
logger.info(` ERR: ${error.message}`)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const _programs = new Collection<epgGrabber.Program>(channelPrograms).map<Program>(
|
||||
program => new Program(program.toObject())
|
||||
)
|
||||
|
||||
programs.concat(_programs)
|
||||
})
|
||||
)
|
||||
|
||||
await Promise.all(requests)
|
||||
|
||||
const output = globalConfig.output || defaultConfig.output
|
||||
|
||||
const pathTemplate = new Template(output)
|
||||
|
||||
const channelsGroupedByKey = channels
|
||||
.uniqBy((channel: Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`)
|
||||
.groupBy((channel: Channel) => {
|
||||
return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' })
|
||||
})
|
||||
|
||||
const programsGroupedByKey = programs
|
||||
.sortBy([(program: Program) => program.channel, (program: Program) => program.start])
|
||||
.groupBy((program: Program) => {
|
||||
const lang =
|
||||
program.titles && program.titles.length && program.titles[0].lang
|
||||
? program.titles[0].lang
|
||||
: 'en'
|
||||
|
||||
return pathTemplate.format({ lang, site: program.site || '' })
|
||||
})
|
||||
|
||||
const gzip = globalConfig.gzip || defaultConfig.gzip
|
||||
const json = globalConfig.json || defaultConfig.json
|
||||
|
||||
for (const groupKey of channelsGroupedByKey.keys()) {
|
||||
const groupChannels = new Collection(channelsGroupedByKey.get(groupKey))
|
||||
const groupPrograms = new Collection(programsGroupedByKey.get(groupKey))
|
||||
const guide = new Guide({
|
||||
filepath: groupKey,
|
||||
gzip,
|
||||
json,
|
||||
channels: groupChannels,
|
||||
programs: groupPrograms
|
||||
})
|
||||
|
||||
await guide.save({ logger })
|
||||
}
|
||||
|
||||
logger.success(` done in ${timer.format('HH[h] mm[m] ss[s]')}`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function getLogoForChannel(channel: Channel): string | null {
|
||||
const feedData = data.feedsKeyByStreamId.get(channel.xmltv_id)
|
||||
if (feedData) {
|
||||
const firstLogo = feedData.getLogos().first()
|
||||
if (firstLogo) return firstLogo.url
|
||||
}
|
||||
|
||||
const [channelId] = channel.xmltv_id.split('@')
|
||||
const channelData = data.channelsKeyById.get(channelId)
|
||||
if (channelData) {
|
||||
const firstLogo = channelData.getLogos().first()
|
||||
if (firstLogo) return firstLogo.url
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function getCircularReplacer() {
|
||||
const seen = new WeakSet()
|
||||
return (key: string, value: any) => {
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
if (seen.has(value)) {
|
||||
return '[Circular]'
|
||||
}
|
||||
seen.add(value)
|
||||
}
|
||||
return value
|
||||
}
|
||||
}
|
||||
@@ -1,88 +1,88 @@
|
||||
import { SITES_DIR, API_DIR, DATA_DIR } from '../../constants'
|
||||
import { Logger, Collection } from '@freearhey/core'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Channel, Worker, WorkerData } from '../../models'
|
||||
import path from 'path'
|
||||
import { ApiGuide } from '../../models/apiGuide'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.start('staring...')
|
||||
|
||||
logger.info('loading channels...')
|
||||
const channels = await loadChannels()
|
||||
logger.info(`found ${channels.count()} channel(s)`)
|
||||
|
||||
logger.info('loading workers.json...')
|
||||
const workers = await loadWorkers()
|
||||
const guidesMap = workers.reduce((acc, data: WorkerData) => {
|
||||
const worker = new Worker(data)
|
||||
if (!worker.channels) return acc
|
||||
worker.channels.forEach(channel => {
|
||||
const [channelId, feedId] = channel.xmltv_id.split('@')
|
||||
const key = [channelId, feedId, channel.site, channel.site_id, channel.lang].join('_')
|
||||
acc[key] = worker.getGuideSources()
|
||||
})
|
||||
return acc
|
||||
}, {})
|
||||
|
||||
logger.info('preparing output...')
|
||||
const output = channels.map((channel: Channel) => {
|
||||
const [channelId, feedId] = channel.xmltv_id.split('@')
|
||||
const key = [channelId, feedId, channel.site, channel.site_id, channel.lang].join('_')
|
||||
|
||||
return new ApiGuide({
|
||||
channel: channelId,
|
||||
feed: feedId,
|
||||
site: channel.site,
|
||||
site_id: channel.site_id,
|
||||
site_name: channel.name,
|
||||
lang: channel.lang,
|
||||
sources: guidesMap[key] || []
|
||||
})
|
||||
})
|
||||
|
||||
logger.info('saving output...')
|
||||
const outputFilename = 'guides.json'
|
||||
await saveOutput(outputFilename, output.toJSON())
|
||||
|
||||
logger.info(`saved to "${path.join(API_DIR, outputFilename)}"`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
async function saveOutput(filename: string, json: string) {
|
||||
const apiStorage = new Storage(API_DIR)
|
||||
await apiStorage.save(filename, json)
|
||||
}
|
||||
|
||||
async function loadWorkers() {
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
|
||||
const workers = await dataStorage.json('workers.json')
|
||||
|
||||
return Array.isArray(workers) ? workers : []
|
||||
}
|
||||
|
||||
async function loadChannels() {
|
||||
const sitesStorage = new Storage(SITES_DIR)
|
||||
|
||||
const files: string[] = await sitesStorage.list('**/*.channels.xml')
|
||||
|
||||
const channels = new Collection<Channel>()
|
||||
for (const filepath of files) {
|
||||
const xml = await sitesStorage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
const channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
channelsFromXML.forEach((channel: Channel) => {
|
||||
channels.add(channel)
|
||||
})
|
||||
}
|
||||
|
||||
return channels
|
||||
}
|
||||
import { SITES_DIR, API_DIR, DATA_DIR } from '../../constants'
|
||||
import { Logger, Collection } from '@freearhey/core'
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Channel, Worker, WorkerData } from '../../models'
|
||||
import path from 'path'
|
||||
import { ApiGuide } from '../../models/apiGuide'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.start('staring...')
|
||||
|
||||
logger.info('loading channels...')
|
||||
const channels = await loadChannels()
|
||||
logger.info(`found ${channels.count()} channel(s)`)
|
||||
|
||||
logger.info('loading workers.json...')
|
||||
const workers = await loadWorkers()
|
||||
const guidesMap = workers.reduce((acc, data: WorkerData) => {
|
||||
const worker = new Worker(data)
|
||||
if (!worker.channels) return acc
|
||||
worker.channels.forEach(channel => {
|
||||
const [channelId, feedId] = channel.xmltv_id.split('@')
|
||||
const key = [channelId, feedId, channel.site, channel.site_id, channel.lang].join('_')
|
||||
acc[key] = worker.getGuideSources()
|
||||
})
|
||||
return acc
|
||||
}, {})
|
||||
|
||||
logger.info('preparing output...')
|
||||
const output = channels.map((channel: Channel) => {
|
||||
const [channelId, feedId] = channel.xmltv_id.split('@')
|
||||
const key = [channelId, feedId, channel.site, channel.site_id, channel.lang].join('_')
|
||||
|
||||
return new ApiGuide({
|
||||
channel: channelId,
|
||||
feed: feedId,
|
||||
site: channel.site,
|
||||
site_id: channel.site_id,
|
||||
site_name: channel.name,
|
||||
lang: channel.lang,
|
||||
sources: guidesMap[key] || []
|
||||
})
|
||||
})
|
||||
|
||||
logger.info('saving output...')
|
||||
const outputFilename = 'guides.json'
|
||||
await saveOutput(outputFilename, output.toJSON())
|
||||
|
||||
logger.info(`saved to "${path.join(API_DIR, outputFilename)}"`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
async function saveOutput(filename: string, json: string) {
|
||||
const apiStorage = new Storage(API_DIR)
|
||||
await apiStorage.save(filename, json)
|
||||
}
|
||||
|
||||
async function loadWorkers() {
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
|
||||
const workers = await dataStorage.json('workers.json')
|
||||
|
||||
return Array.isArray(workers) ? workers : []
|
||||
}
|
||||
|
||||
async function loadChannels() {
|
||||
const sitesStorage = new Storage(SITES_DIR)
|
||||
|
||||
const files: string[] = await sitesStorage.list('**/*.channels.xml')
|
||||
|
||||
const channels = new Collection<Channel>()
|
||||
for (const filepath of files) {
|
||||
const xml = await sitesStorage.load(filepath)
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(xml)
|
||||
const channelsFromXML = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
channelsFromXML.forEach((channel: Channel) => {
|
||||
channels.add(channel)
|
||||
})
|
||||
}
|
||||
|
||||
return channels
|
||||
}
|
||||
@@ -1,58 +1,58 @@
|
||||
import { HTMLTableRow, HTMLTableDataItem, HTMLTableColumn } from '../../types/htmlTable'
|
||||
import { Worker, WorkerData, WorkerGuideSource } from '../../models'
|
||||
import { DATA_DIR, ROOT_DIR } from '../../constants'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Collection } from '@freearhey/core'
|
||||
import { Logger } from '@freearhey/core'
|
||||
import { HTMLTable } from '../../core'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger({ level: process.env.NODE_ENV === 'test' ? -999 : 3 })
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
|
||||
logger.info('loading workers.json...')
|
||||
const workers = await dataStorage.json('workers.json')
|
||||
|
||||
if (!Array.isArray(workers)) return
|
||||
|
||||
logger.info('creating guides table...')
|
||||
const rows = new Collection<HTMLTableRow>()
|
||||
workers.forEach((data: WorkerData) => {
|
||||
const worker = new Worker(data)
|
||||
|
||||
const sources = worker.getGuideSources()
|
||||
rows.add(
|
||||
new Collection<HTMLTableDataItem>([
|
||||
{ value: worker.host },
|
||||
{ value: worker.getStatusEmoji(), align: 'center' },
|
||||
{ value: worker.getChannelsCount().toString(), align: 'right' },
|
||||
{ value: worker.getLastUpdated(), align: 'left' },
|
||||
{
|
||||
value: sources.length
|
||||
? sources
|
||||
.map((source: WorkerGuideSource) => `<a href="${source.url}">${source.format}</a>`)
|
||||
.join(' | ')
|
||||
: '-'
|
||||
}
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
logger.info('updating guides.md...')
|
||||
const rootStorage = new Storage(ROOT_DIR)
|
||||
const table = new HTMLTable(
|
||||
rows,
|
||||
new Collection<HTMLTableColumn>([
|
||||
{ name: 'Host', align: 'left' },
|
||||
{ name: 'Status', align: 'left' },
|
||||
{ name: 'Channels', align: 'left' },
|
||||
{ name: 'Last Updated', align: 'left' },
|
||||
{ name: 'Links', align: 'left' }
|
||||
])
|
||||
)
|
||||
const guidesTemplate = await new Storage().load('scripts/templates/_guides.md')
|
||||
const guidesContent = guidesTemplate.replace('_TABLE_', table.toString())
|
||||
await rootStorage.save('GUIDES.md', guidesContent)
|
||||
}
|
||||
|
||||
main()
|
||||
import { HTMLTableRow, HTMLTableDataItem, HTMLTableColumn } from '../../types/htmlTable'
|
||||
import { Worker, WorkerData, WorkerGuideSource } from '../../models'
|
||||
import { DATA_DIR, ROOT_DIR } from '../../constants'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Collection } from '@freearhey/core'
|
||||
import { Logger } from '@freearhey/core'
|
||||
import { HTMLTable } from '../../core'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger({ level: process.env.NODE_ENV === 'test' ? -999 : 3 })
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
|
||||
logger.info('loading workers.json...')
|
||||
const workers = await dataStorage.json('workers.json')
|
||||
|
||||
if (!Array.isArray(workers)) return
|
||||
|
||||
logger.info('creating guides table...')
|
||||
const rows = new Collection<HTMLTableRow>()
|
||||
workers.forEach((data: WorkerData) => {
|
||||
const worker = new Worker(data)
|
||||
|
||||
const sources = worker.getGuideSources()
|
||||
rows.add(
|
||||
new Collection<HTMLTableDataItem>([
|
||||
{ value: worker.host },
|
||||
{ value: worker.getStatusEmoji(), align: 'center' },
|
||||
{ value: worker.getChannelsCount().toString(), align: 'right' },
|
||||
{ value: worker.getLastUpdated(), align: 'left' },
|
||||
{
|
||||
value: sources.length
|
||||
? sources
|
||||
.map((source: WorkerGuideSource) => `<a href="${source.url}">${source.format}</a>`)
|
||||
.join(' | ')
|
||||
: '-'
|
||||
}
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
logger.info('updating guides.md...')
|
||||
const rootStorage = new Storage(ROOT_DIR)
|
||||
const table = new HTMLTable(
|
||||
rows,
|
||||
new Collection<HTMLTableColumn>([
|
||||
{ name: 'Host', align: 'left' },
|
||||
{ name: 'Status', align: 'left' },
|
||||
{ name: 'Channels', align: 'left' },
|
||||
{ name: 'Last Updated', align: 'left' },
|
||||
{ name: 'Links', align: 'left' }
|
||||
])
|
||||
)
|
||||
const guidesTemplate = await new Storage().load('scripts/templates/_guides.md')
|
||||
const guidesContent = guidesTemplate.replace('_TABLE_', table.toString())
|
||||
await rootStorage.save('GUIDES.md', guidesContent)
|
||||
}
|
||||
|
||||
main()
|
||||
@@ -1,46 +1,46 @@
|
||||
import { SITES_DIR, EOL } from '../../constants'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Logger } from '@freearhey/core'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { program } from 'commander'
|
||||
import fs from 'fs-extra'
|
||||
|
||||
program.argument('<site>', 'Domain name of the site').parse(process.argv)
|
||||
|
||||
const domain = program.args[0]
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage(SITES_DIR)
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info(`Initializing "${domain}"...${EOL}`)
|
||||
|
||||
const dir = domain
|
||||
if (await storage.exists(dir)) {
|
||||
throw new Error(`Folder "${dir}" already exists`)
|
||||
}
|
||||
|
||||
await storage.createDir(dir)
|
||||
|
||||
logger.info(`Creating "${dir}/${domain}.test.js"...`)
|
||||
const testTemplate = fs.readFileSync(pathToFileURL('scripts/templates/_test.js'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
await storage.save(`${dir}/${domain}.test.js`, testTemplate.replace(/<DOMAIN>/g, domain))
|
||||
|
||||
logger.info(`Creating "${dir}/${domain}.config.js"...`)
|
||||
const configTemplate = fs.readFileSync(pathToFileURL('scripts/templates/_config.js'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
await storage.save(`${dir}/${domain}.config.js`, configTemplate.replace(/<DOMAIN>/g, domain))
|
||||
|
||||
logger.info(`Creating "${dir}/readme.md"...`)
|
||||
const readmeTemplate = fs.readFileSync(pathToFileURL('scripts/templates/_readme.md'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
await storage.save(`${dir}/readme.md`, readmeTemplate.replace(/<DOMAIN>/g, domain))
|
||||
|
||||
logger.info(`${EOL}Done`)
|
||||
}
|
||||
|
||||
main()
|
||||
import { SITES_DIR, EOL } from '../../constants'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Logger } from '@freearhey/core'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { program } from 'commander'
|
||||
import fs from 'fs-extra'
|
||||
|
||||
program.argument('<site>', 'Domain name of the site').parse(process.argv)
|
||||
|
||||
const domain = program.args[0]
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage(SITES_DIR)
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info(`Initializing "${domain}"...${EOL}`)
|
||||
|
||||
const dir = domain
|
||||
if (await storage.exists(dir)) {
|
||||
throw new Error(`Folder "${dir}" already exists`)
|
||||
}
|
||||
|
||||
await storage.createDir(dir)
|
||||
|
||||
logger.info(`Creating "${dir}/${domain}.test.js"...`)
|
||||
const testTemplate = fs.readFileSync(pathToFileURL('scripts/templates/_test.js'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
await storage.save(`${dir}/${domain}.test.js`, testTemplate.replace(/<DOMAIN>/g, domain))
|
||||
|
||||
logger.info(`Creating "${dir}/${domain}.config.js"...`)
|
||||
const configTemplate = fs.readFileSync(pathToFileURL('scripts/templates/_config.js'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
await storage.save(`${dir}/${domain}.config.js`, configTemplate.replace(/<DOMAIN>/g, domain))
|
||||
|
||||
logger.info(`Creating "${dir}/readme.md"...`)
|
||||
const readmeTemplate = fs.readFileSync(pathToFileURL('scripts/templates/_readme.md'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
await storage.save(`${dir}/readme.md`, readmeTemplate.replace(/<DOMAIN>/g, domain))
|
||||
|
||||
logger.info(`${EOL}Done`)
|
||||
}
|
||||
|
||||
main()
|
||||
@@ -1,84 +1,84 @@
|
||||
import { HTMLTableDataItem, HTMLTableRow, HTMLTableColumn } from '../../types/htmlTable'
|
||||
import { SITES_DIR, ROOT_DIR } from '../../constants'
|
||||
import { Logger, Collection } from '@freearhey/core'
|
||||
import { Issue, Site, Channel } from '../../models'
|
||||
import { HTMLTable, loadIssues } from '../../core'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import * as epgGrabber from 'epg-grabber'
|
||||
import { EPGGrabber } from 'epg-grabber'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger({ level: -999 })
|
||||
const sitesStorage = new Storage(SITES_DIR)
|
||||
const sites = new Collection<Site>()
|
||||
|
||||
logger.info('loading list of sites')
|
||||
const folders = await sitesStorage.list('*/')
|
||||
|
||||
logger.info('loading issues...')
|
||||
const issues = await loadIssues()
|
||||
|
||||
logger.info('putting the data together...')
|
||||
const brokenGuideReports = issues.filter(issue =>
|
||||
issue.labels.find((label: string) => label === 'broken guide')
|
||||
)
|
||||
for (const domain of folders) {
|
||||
const filteredIssues = brokenGuideReports.filter(
|
||||
(issue: Issue) => domain === issue.data.get('site')
|
||||
)
|
||||
|
||||
const site = new Site({
|
||||
domain,
|
||||
issues: filteredIssues
|
||||
})
|
||||
|
||||
const files = await sitesStorage.list(`${domain}/*.channels.xml`)
|
||||
for (const filepath of files) {
|
||||
const xml = await sitesStorage.load(filepath)
|
||||
const channelsFromXML = EPGGrabber.parseChannelsXML(xml)
|
||||
const channels = new Collection(channelsFromXML).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
site.channels = site.channels.concat(channels)
|
||||
}
|
||||
|
||||
sites.add(site)
|
||||
}
|
||||
|
||||
logger.info('creating sites table...')
|
||||
const rows = new Collection<HTMLTableRow>()
|
||||
sites.forEach((site: Site) => {
|
||||
rows.add(
|
||||
new Collection<HTMLTableDataItem>([
|
||||
{ value: `<a href="sites/${site.domain}">${site.domain}</a>` },
|
||||
{
|
||||
value: site.channels
|
||||
.uniqBy((channel: Channel) => channel.site_id)
|
||||
.count()
|
||||
.toString(),
|
||||
align: 'right'
|
||||
},
|
||||
{ value: site.getStatus().emoji, align: 'center' },
|
||||
{ value: site.getIssueUrls().all().join(', ') }
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
logger.info('updating sites.md...')
|
||||
const table = new HTMLTable(
|
||||
rows,
|
||||
new Collection<HTMLTableColumn>([
|
||||
{ name: 'Site', align: 'left' },
|
||||
{ name: 'Channels', align: 'left' },
|
||||
{ name: 'Status', align: 'left' },
|
||||
{ name: 'Notes', align: 'left' }
|
||||
])
|
||||
)
|
||||
const rootStorage = new Storage(ROOT_DIR)
|
||||
const sitesTemplate = await new Storage().load('scripts/templates/_sites.md')
|
||||
const sitesContent = sitesTemplate.replace('_TABLE_', table.toString())
|
||||
await rootStorage.save('SITES.md', sitesContent)
|
||||
}
|
||||
|
||||
main()
|
||||
import { HTMLTableDataItem, HTMLTableRow, HTMLTableColumn } from '../../types/htmlTable'
|
||||
import { SITES_DIR, ROOT_DIR } from '../../constants'
|
||||
import { Logger, Collection } from '@freearhey/core'
|
||||
import { Issue, Site, Channel } from '../../models'
|
||||
import { HTMLTable, loadIssues } from '../../core'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import * as epgGrabber from 'epg-grabber'
|
||||
import { EPGGrabber } from 'epg-grabber'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger({ level: -999 })
|
||||
const sitesStorage = new Storage(SITES_DIR)
|
||||
const sites = new Collection<Site>()
|
||||
|
||||
logger.info('loading list of sites')
|
||||
const folders = await sitesStorage.list('*/')
|
||||
|
||||
logger.info('loading issues...')
|
||||
const issues = await loadIssues()
|
||||
|
||||
logger.info('putting the data together...')
|
||||
const brokenGuideReports = issues.filter(issue =>
|
||||
issue.labels.find((label: string) => label === 'broken guide')
|
||||
)
|
||||
for (const domain of folders) {
|
||||
const filteredIssues = brokenGuideReports.filter(
|
||||
(issue: Issue) => domain === issue.data.get('site')
|
||||
)
|
||||
|
||||
const site = new Site({
|
||||
domain,
|
||||
issues: filteredIssues
|
||||
})
|
||||
|
||||
const files = await sitesStorage.list(`${domain}/*.channels.xml`)
|
||||
for (const filepath of files) {
|
||||
const xml = await sitesStorage.load(filepath)
|
||||
const channelsFromXML = EPGGrabber.parseChannelsXML(xml)
|
||||
const channels = new Collection(channelsFromXML).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
site.channels = site.channels.concat(channels)
|
||||
}
|
||||
|
||||
sites.add(site)
|
||||
}
|
||||
|
||||
logger.info('creating sites table...')
|
||||
const rows = new Collection<HTMLTableRow>()
|
||||
sites.forEach((site: Site) => {
|
||||
rows.add(
|
||||
new Collection<HTMLTableDataItem>([
|
||||
{ value: `<a href="sites/${site.domain}">${site.domain}</a>` },
|
||||
{
|
||||
value: site.channels
|
||||
.uniqBy((channel: Channel) => channel.site_id)
|
||||
.count()
|
||||
.toString(),
|
||||
align: 'right'
|
||||
},
|
||||
{ value: site.getStatus().emoji, align: 'center' },
|
||||
{ value: site.getIssueUrls().all().join(', ') }
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
logger.info('updating sites.md...')
|
||||
const table = new HTMLTable(
|
||||
rows,
|
||||
new Collection<HTMLTableColumn>([
|
||||
{ name: 'Site', align: 'left' },
|
||||
{ name: 'Channels', align: 'left' },
|
||||
{ name: 'Status', align: 'left' },
|
||||
{ name: 'Notes', align: 'left' }
|
||||
])
|
||||
)
|
||||
const rootStorage = new Storage(ROOT_DIR)
|
||||
const sitesTemplate = await new Storage().load('scripts/templates/_sites.md')
|
||||
const sitesContent = sitesTemplate.replace('_TABLE_', table.toString())
|
||||
await rootStorage.save('SITES.md', sitesContent)
|
||||
}
|
||||
|
||||
main()
|
||||
+122
-122
@@ -1,122 +1,122 @@
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { DATA_DIR, ROOT_DIR } from '../../constants'
|
||||
import AxiosMockAdapter from 'axios-mock-adapter'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Channel, Worker } from '../../models'
|
||||
import { Collection } from '@freearhey/core'
|
||||
import { Logger } from '@freearhey/core'
|
||||
import epgParser from 'epg-parser'
|
||||
import axios from 'axios'
|
||||
import path from 'path'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger({ level: process.env.NODE_ENV === 'test' ? -999 : 3 })
|
||||
const rootStorage = new Storage(ROOT_DIR)
|
||||
const workers = new Map<string, Worker>()
|
||||
|
||||
logger.info('loading workers.txt...')
|
||||
const workersTxt = await rootStorage.load('workers.txt')
|
||||
const hosts = workersTxt.split('\r\n')
|
||||
|
||||
hosts.forEach((host: string) => {
|
||||
if (!host) return
|
||||
|
||||
const worker = new Worker({ host })
|
||||
|
||||
workers.set(host, worker)
|
||||
})
|
||||
|
||||
for (const worker of workers.values()) {
|
||||
logger.info(`processing "${worker.host}"...`)
|
||||
|
||||
const client = axios.create({
|
||||
baseURL: worker.getBaseUrl(),
|
||||
timeout: 60000
|
||||
})
|
||||
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
const mock = new AxiosMockAdapter(client)
|
||||
if (worker.host === 'example.com') {
|
||||
mock.onGet('worker.json').reply(404)
|
||||
} else {
|
||||
const testStorage = new Storage('tests/__data__/input/workers_load')
|
||||
mock.onGet('worker.json').reply(200, await testStorage.load('worker.json'))
|
||||
mock.onGet('channels.xml').reply(200, await testStorage.load('channels.xml'))
|
||||
mock.onGet('guide.xml').reply(200, await testStorage.load('guide.xml'))
|
||||
}
|
||||
}
|
||||
|
||||
const workerConfig = await client
|
||||
.get('worker.json')
|
||||
.then(res => res.data)
|
||||
.catch(err => {
|
||||
worker.setStatus(err.status)
|
||||
logger.error(err.message)
|
||||
})
|
||||
|
||||
if (!workerConfig) {
|
||||
worker.setStatus('MISSING_WORKER_CONFIG')
|
||||
logger.error('Unable to load "worker.json"')
|
||||
continue
|
||||
}
|
||||
|
||||
worker
|
||||
.setChannelsPath(workerConfig.channels)
|
||||
.setGuideXmlPath(
|
||||
typeof workerConfig.guide === 'string' ? workerConfig.guide : workerConfig?.guide?.xml
|
||||
)
|
||||
.setGuideGzipPath(workerConfig?.guide?.gzip)
|
||||
.setGuideJsonPath(workerConfig?.guide?.json)
|
||||
|
||||
if (!worker.channelsPath) {
|
||||
worker.setStatus('MISSING_CHANNELS_PATH')
|
||||
logger.error('The "channels" property is missing from the workers config')
|
||||
continue
|
||||
}
|
||||
|
||||
if (!worker.guideXmlPath) {
|
||||
worker.setStatus('MISSING_GUIDE_XML_PATH')
|
||||
logger.error('The "guide" property is missing from the workers config')
|
||||
continue
|
||||
}
|
||||
|
||||
const channelsXml = await client
|
||||
.get(worker.channelsPath)
|
||||
.then(res => res.data)
|
||||
.catch(err => {
|
||||
worker.status = err.status
|
||||
logger.error(err.message)
|
||||
})
|
||||
|
||||
if (!channelsXml) continue
|
||||
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(channelsXml)
|
||||
worker.channels = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
const guideXml = await client
|
||||
.get(worker.guideXmlPath)
|
||||
.then(res => res.data)
|
||||
.catch(err => {
|
||||
worker.setStatus(err.status)
|
||||
logger.error(err.message)
|
||||
})
|
||||
|
||||
if (!guideXml) continue
|
||||
|
||||
const parsedGuide = epgParser.parse(guideXml)
|
||||
worker.lastUpdated = parsedGuide.date
|
||||
|
||||
worker.setStatus('OK')
|
||||
}
|
||||
|
||||
const output = [...workers.values()]
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const outputFilename = 'workers.json'
|
||||
await dataStorage.save(outputFilename, JSON.stringify(output))
|
||||
|
||||
logger.info(`saved to "${path.join(DATA_DIR, outputFilename)}"`)
|
||||
}
|
||||
|
||||
main()
|
||||
import epgGrabber, { EPGGrabber } from 'epg-grabber'
|
||||
import { DATA_DIR, ROOT_DIR } from '../../constants'
|
||||
import AxiosMockAdapter from 'axios-mock-adapter'
|
||||
import { Storage } from '@freearhey/storage-js'
|
||||
import { Channel, Worker } from '../../models'
|
||||
import { Collection } from '@freearhey/core'
|
||||
import { Logger } from '@freearhey/core'
|
||||
import epgParser from 'epg-parser'
|
||||
import axios from 'axios'
|
||||
import path from 'path'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger({ level: process.env.NODE_ENV === 'test' ? -999 : 3 })
|
||||
const rootStorage = new Storage(ROOT_DIR)
|
||||
const workers = new Map<string, Worker>()
|
||||
|
||||
logger.info('loading workers.txt...')
|
||||
const workersTxt = await rootStorage.load('workers.txt')
|
||||
const hosts = workersTxt.split('\r\n')
|
||||
|
||||
hosts.forEach((host: string) => {
|
||||
if (!host) return
|
||||
|
||||
const worker = new Worker({ host })
|
||||
|
||||
workers.set(host, worker)
|
||||
})
|
||||
|
||||
for (const worker of workers.values()) {
|
||||
logger.info(`processing "${worker.host}"...`)
|
||||
|
||||
const client = axios.create({
|
||||
baseURL: worker.getBaseUrl(),
|
||||
timeout: 60000
|
||||
})
|
||||
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
const mock = new AxiosMockAdapter(client)
|
||||
if (worker.host === 'example.com') {
|
||||
mock.onGet('worker.json').reply(404)
|
||||
} else {
|
||||
const testStorage = new Storage('tests/__data__/input/workers_load')
|
||||
mock.onGet('worker.json').reply(200, await testStorage.load('worker.json'))
|
||||
mock.onGet('channels.xml').reply(200, await testStorage.load('channels.xml'))
|
||||
mock.onGet('guide.xml').reply(200, await testStorage.load('guide.xml'))
|
||||
}
|
||||
}
|
||||
|
||||
const workerConfig = await client
|
||||
.get('worker.json')
|
||||
.then(res => res.data)
|
||||
.catch(err => {
|
||||
worker.setStatus(err.status)
|
||||
logger.error(err.message)
|
||||
})
|
||||
|
||||
if (!workerConfig) {
|
||||
worker.setStatus('MISSING_WORKER_CONFIG')
|
||||
logger.error('Unable to load "worker.json"')
|
||||
continue
|
||||
}
|
||||
|
||||
worker
|
||||
.setChannelsPath(workerConfig.channels)
|
||||
.setGuideXmlPath(
|
||||
typeof workerConfig.guide === 'string' ? workerConfig.guide : workerConfig?.guide?.xml
|
||||
)
|
||||
.setGuideGzipPath(workerConfig?.guide?.gzip)
|
||||
.setGuideJsonPath(workerConfig?.guide?.json)
|
||||
|
||||
if (!worker.channelsPath) {
|
||||
worker.setStatus('MISSING_CHANNELS_PATH')
|
||||
logger.error('The "channels" property is missing from the workers config')
|
||||
continue
|
||||
}
|
||||
|
||||
if (!worker.guideXmlPath) {
|
||||
worker.setStatus('MISSING_GUIDE_XML_PATH')
|
||||
logger.error('The "guide" property is missing from the workers config')
|
||||
continue
|
||||
}
|
||||
|
||||
const channelsXml = await client
|
||||
.get(worker.channelsPath)
|
||||
.then(res => res.data)
|
||||
.catch(err => {
|
||||
worker.status = err.status
|
||||
logger.error(err.message)
|
||||
})
|
||||
|
||||
if (!channelsXml) continue
|
||||
|
||||
const parsedChannels = EPGGrabber.parseChannelsXML(channelsXml)
|
||||
worker.channels = new Collection(parsedChannels).map(
|
||||
(channel: epgGrabber.Channel) => new Channel(channel.toObject())
|
||||
)
|
||||
|
||||
const guideXml = await client
|
||||
.get(worker.guideXmlPath)
|
||||
.then(res => res.data)
|
||||
.catch(err => {
|
||||
worker.setStatus(err.status)
|
||||
logger.error(err.message)
|
||||
})
|
||||
|
||||
if (!guideXml) continue
|
||||
|
||||
const parsedGuide = epgParser.parse(guideXml)
|
||||
worker.lastUpdated = parsedGuide.date
|
||||
|
||||
worker.setStatus('OK')
|
||||
}
|
||||
|
||||
const output = [...workers.values()]
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const outputFilename = 'workers.json'
|
||||
await dataStorage.save(outputFilename, JSON.stringify(output))
|
||||
|
||||
logger.info(`saved to "${path.join(DATA_DIR, outputFilename)}"`)
|
||||
}
|
||||
|
||||
main()
|
||||
Reference in new issue
Block a user