mirror of
https://github.com/iptv-org/epg
synced 2025-12-17 19:06:58 -05:00
use splitted lodash modules for better efficiency
This commit is contained in:
theofficialomega
@@ -1,41 +1,41 @@
|
||||
import { Logger, Collection, Storage } from '@freearhey/core'
|
||||
import { SITES_DIR, API_DIR } from '../../constants'
|
||||
import { GuideChannel } from '../../models'
|
||||
import { ChannelsParser } from '../../core'
|
||||
import epgGrabber from 'epg-grabber'
|
||||
import path from 'path'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.start('staring...')
|
||||
|
||||
logger.info('loading channels...')
|
||||
const sitesStorage = new Storage(SITES_DIR)
|
||||
const parser = new ChannelsParser({
|
||||
storage: sitesStorage
|
||||
})
|
||||
|
||||
const files: string[] = await sitesStorage.list('**/*.channels.xml')
|
||||
|
||||
const channels = new Collection()
|
||||
for (const filepath of files) {
|
||||
const channelList = await parser.parse(filepath)
|
||||
|
||||
channelList.channels.forEach((data: epgGrabber.Channel) => {
|
||||
channels.add(new GuideChannel(data))
|
||||
})
|
||||
}
|
||||
|
||||
logger.info(`found ${channels.count()} channel(s)`)
|
||||
|
||||
const output = channels.map((channel: GuideChannel) => channel.toJSON())
|
||||
|
||||
const apiStorage = new Storage(API_DIR)
|
||||
const outputFilename = 'guides.json'
|
||||
await apiStorage.save('guides.json', output.toJSON())
|
||||
|
||||
logger.info(`saved to "${path.join(API_DIR, outputFilename)}"`)
|
||||
}
|
||||
|
||||
main()
|
||||
import { Logger, Collection, Storage } from '@freearhey/core'
|
||||
import { SITES_DIR, API_DIR } from '../../constants'
|
||||
import { GuideChannel } from '../../models'
|
||||
import { ChannelsParser } from '../../core'
|
||||
import epgGrabber from 'epg-grabber'
|
||||
import path from 'path'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.start('staring...')
|
||||
|
||||
logger.info('loading channels...')
|
||||
const sitesStorage = new Storage(SITES_DIR)
|
||||
const parser = new ChannelsParser({
|
||||
storage: sitesStorage
|
||||
})
|
||||
|
||||
const files: string[] = await sitesStorage.list('**/*.channels.xml')
|
||||
|
||||
const channels = new Collection()
|
||||
for (const filepath of files) {
|
||||
const channelList = await parser.parse(filepath)
|
||||
|
||||
channelList.channels.forEach((data: epgGrabber.Channel) => {
|
||||
channels.add(new GuideChannel(data))
|
||||
})
|
||||
}
|
||||
|
||||
logger.info(`found ${channels.count()} channel(s)`)
|
||||
|
||||
const output = channels.map((channel: GuideChannel) => channel.toJSON())
|
||||
|
||||
const apiStorage = new Storage(API_DIR)
|
||||
const outputFilename = 'guides.json'
|
||||
await apiStorage.save('guides.json', output.toJSON())
|
||||
|
||||
logger.info(`saved to "${path.join(API_DIR, outputFilename)}"`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
@@ -1,25 +1,25 @@
|
||||
import { DATA_DIR } from '../../constants'
|
||||
import { Storage } from '@freearhey/core'
|
||||
import { DataLoader } from '../../core'
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage(DATA_DIR)
|
||||
const loader = new DataLoader({ storage })
|
||||
|
||||
await Promise.all([
|
||||
loader.download('blocklist.json'),
|
||||
loader.download('categories.json'),
|
||||
loader.download('channels.json'),
|
||||
loader.download('countries.json'),
|
||||
loader.download('languages.json'),
|
||||
loader.download('regions.json'),
|
||||
loader.download('subdivisions.json'),
|
||||
loader.download('feeds.json'),
|
||||
loader.download('timezones.json'),
|
||||
loader.download('guides.json'),
|
||||
loader.download('streams.json'),
|
||||
loader.download('logos.json')
|
||||
])
|
||||
}
|
||||
|
||||
main()
|
||||
import { DATA_DIR } from '../../constants'
|
||||
import { Storage } from '@freearhey/core'
|
||||
import { DataLoader } from '../../core'
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage(DATA_DIR)
|
||||
const loader = new DataLoader({ storage })
|
||||
|
||||
await Promise.all([
|
||||
loader.download('blocklist.json'),
|
||||
loader.download('categories.json'),
|
||||
loader.download('channels.json'),
|
||||
loader.download('countries.json'),
|
||||
loader.download('languages.json'),
|
||||
loader.download('regions.json'),
|
||||
loader.download('subdivisions.json'),
|
||||
loader.download('feeds.json'),
|
||||
loader.download('timezones.json'),
|
||||
loader.download('guides.json'),
|
||||
loader.download('streams.json'),
|
||||
loader.download('logos.json')
|
||||
])
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
@@ -1,109 +1,109 @@
|
||||
import chalk from 'chalk'
|
||||
import { program } from 'commander'
|
||||
import { Storage, File } from '@freearhey/core'
|
||||
import { XmlDocument, XsdValidator, XmlValidateError, ErrorDetail } from 'libxml2-wasm'
|
||||
|
||||
const xsd = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
|
||||
<xs:element name="channels">
|
||||
<xs:complexType>
|
||||
<xs:sequence>
|
||||
<xs:element minOccurs="0" maxOccurs="unbounded" ref="channel"/>
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:element name="channel">
|
||||
<xs:complexType mixed="true">
|
||||
<xs:attribute use="required" ref="site"/>
|
||||
<xs:attribute use="required" ref="lang"/>
|
||||
<xs:attribute use="required" ref="site_id"/>
|
||||
<xs:attribute name="xmltv_id" use="required" type="xs:string"/>
|
||||
<xs:attribute name="logo" type="xs:string"/>
|
||||
<xs:attribute name="lcn" type="xs:string"/>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:attribute name="site">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="site_id">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="lang">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
</xs:schema>`
|
||||
|
||||
program.argument('[filepath...]', 'Path to *.channels.xml files to check').parse(process.argv)
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage()
|
||||
|
||||
let errors: ErrorDetail[] = []
|
||||
|
||||
const files = program.args.length ? program.args : await storage.list('sites/**/*.channels.xml')
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'xml') continue
|
||||
|
||||
const xml = await storage.load(filepath)
|
||||
|
||||
let localErrors: ErrorDetail[] = []
|
||||
|
||||
try {
|
||||
const schema = XmlDocument.fromString(xsd)
|
||||
const validator = XsdValidator.fromDoc(schema)
|
||||
const doc = XmlDocument.fromString(xml)
|
||||
|
||||
validator.validate(doc)
|
||||
|
||||
schema.dispose()
|
||||
validator.dispose()
|
||||
doc.dispose()
|
||||
} catch (_error) {
|
||||
const error = _error as XmlValidateError
|
||||
|
||||
localErrors = localErrors.concat(error.details)
|
||||
}
|
||||
|
||||
xml.split('\n').forEach((line: string, lineIndex: number) => {
|
||||
const found = line.match(/='/)
|
||||
if (found) {
|
||||
const colIndex = found.index || 0
|
||||
localErrors.push({
|
||||
line: lineIndex + 1,
|
||||
col: colIndex + 1,
|
||||
message: 'Single quotes cannot be used in attributes'
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if (localErrors.length) {
|
||||
console.log(`\n${chalk.underline(filepath)}`)
|
||||
localErrors.forEach((error: ErrorDetail) => {
|
||||
const position = `${error.line}:${error.col}`
|
||||
console.log(` ${chalk.gray(position.padEnd(4, ' '))} ${error.message.trim()}`)
|
||||
})
|
||||
|
||||
errors = errors.concat(localErrors)
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
console.log(chalk.red(`\n${errors.length} error(s)`))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
import chalk from 'chalk'
|
||||
import { program } from 'commander'
|
||||
import { Storage, File } from '@freearhey/core'
|
||||
import { XmlDocument, XsdValidator, XmlValidateError, ErrorDetail } from 'libxml2-wasm'
|
||||
|
||||
const xsd = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
|
||||
<xs:element name="channels">
|
||||
<xs:complexType>
|
||||
<xs:sequence>
|
||||
<xs:element minOccurs="0" maxOccurs="unbounded" ref="channel"/>
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:element name="channel">
|
||||
<xs:complexType mixed="true">
|
||||
<xs:attribute use="required" ref="site"/>
|
||||
<xs:attribute use="required" ref="lang"/>
|
||||
<xs:attribute use="required" ref="site_id"/>
|
||||
<xs:attribute name="xmltv_id" use="required" type="xs:string"/>
|
||||
<xs:attribute name="logo" type="xs:string"/>
|
||||
<xs:attribute name="lcn" type="xs:string"/>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:attribute name="site">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="site_id">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="lang">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
</xs:schema>`
|
||||
|
||||
program.argument('[filepath...]', 'Path to *.channels.xml files to check').parse(process.argv)
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage()
|
||||
|
||||
let errors: ErrorDetail[] = []
|
||||
|
||||
const files = program.args.length ? program.args : await storage.list('sites/**/*.channels.xml')
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'xml') continue
|
||||
|
||||
const xml = await storage.load(filepath)
|
||||
|
||||
let localErrors: ErrorDetail[] = []
|
||||
|
||||
try {
|
||||
const schema = XmlDocument.fromString(xsd)
|
||||
const validator = XsdValidator.fromDoc(schema)
|
||||
const doc = XmlDocument.fromString(xml)
|
||||
|
||||
validator.validate(doc)
|
||||
|
||||
schema.dispose()
|
||||
validator.dispose()
|
||||
doc.dispose()
|
||||
} catch (_error) {
|
||||
const error = _error as XmlValidateError
|
||||
|
||||
localErrors = localErrors.concat(error.details)
|
||||
}
|
||||
|
||||
xml.split('\n').forEach((line: string, lineIndex: number) => {
|
||||
const found = line.match(/='/)
|
||||
if (found) {
|
||||
const colIndex = found.index || 0
|
||||
localErrors.push({
|
||||
line: lineIndex + 1,
|
||||
col: colIndex + 1,
|
||||
message: 'Single quotes cannot be used in attributes'
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if (localErrors.length) {
|
||||
console.log(`\n${chalk.underline(filepath)}`)
|
||||
localErrors.forEach((error: ErrorDetail) => {
|
||||
const position = `${error.line}:${error.col}`
|
||||
console.log(` ${chalk.gray(position.padEnd(4, ' '))} ${error.message.trim()}`)
|
||||
})
|
||||
|
||||
errors = errors.concat(localErrors)
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
console.log(chalk.red(`\n${errors.length} error(s)`))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
@@ -1,76 +1,76 @@
|
||||
import { IssueLoader, HTMLTable, ChannelsParser } from '../../core'
|
||||
import { Logger, Storage, Collection } from '@freearhey/core'
|
||||
import { ChannelList, Issue, Site } from '../../models'
|
||||
import { SITES_DIR, ROOT_DIR } from '../../constants'
|
||||
import { Channel } from 'epg-grabber'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger({ level: -999 })
|
||||
const issueLoader = new IssueLoader()
|
||||
const sitesStorage = new Storage(SITES_DIR)
|
||||
const sites = new Collection()
|
||||
|
||||
logger.info('loading channels...')
|
||||
const channelsParser = new ChannelsParser({
|
||||
storage: sitesStorage
|
||||
})
|
||||
|
||||
logger.info('loading list of sites')
|
||||
const folders = await sitesStorage.list('*/')
|
||||
|
||||
logger.info('loading issues...')
|
||||
const issues = await issueLoader.load()
|
||||
|
||||
logger.info('putting the data together...')
|
||||
const brokenGuideReports = issues.filter(issue =>
|
||||
issue.labels.find((label: string) => label === 'broken guide')
|
||||
)
|
||||
for (const domain of folders) {
|
||||
const filteredIssues = brokenGuideReports.filter(
|
||||
(issue: Issue) => domain === issue.data.get('site')
|
||||
)
|
||||
|
||||
const site = new Site({
|
||||
domain,
|
||||
issues: filteredIssues
|
||||
})
|
||||
|
||||
const files = await sitesStorage.list(`${domain}/*.channels.xml`)
|
||||
for (const filepath of files) {
|
||||
const channelList: ChannelList = await channelsParser.parse(filepath)
|
||||
|
||||
site.totalChannels += channelList.channels.count()
|
||||
site.markedChannels += channelList.channels
|
||||
.filter((channel: Channel) => channel.xmltv_id)
|
||||
.count()
|
||||
}
|
||||
|
||||
sites.add(site)
|
||||
}
|
||||
|
||||
logger.info('creating sites table...')
|
||||
const tableData = new Collection()
|
||||
sites.forEach((site: Site) => {
|
||||
tableData.add([
|
||||
{ value: `<a href="sites/${site.domain}">${site.domain}</a>` },
|
||||
{ value: site.totalChannels, align: 'right' },
|
||||
{ value: site.markedChannels, align: 'right' },
|
||||
{ value: site.getStatus().emoji, align: 'center' },
|
||||
{ value: site.getIssues().all().join(', ') }
|
||||
])
|
||||
})
|
||||
|
||||
logger.info('updating sites.md...')
|
||||
const table = new HTMLTable(tableData.all(), [
|
||||
{ name: 'Site', align: 'left' },
|
||||
{ name: 'Channels<br>(total / with xmltv-id)', colspan: 2, align: 'left' },
|
||||
{ name: 'Status', align: 'left' },
|
||||
{ name: 'Notes', align: 'left' }
|
||||
])
|
||||
const rootStorage = new Storage(ROOT_DIR)
|
||||
const sitesTemplate = await new Storage().load('scripts/templates/_sites.md')
|
||||
const sitesContent = sitesTemplate.replace('_TABLE_', table.toString())
|
||||
await rootStorage.save('SITES.md', sitesContent)
|
||||
}
|
||||
|
||||
main()
|
||||
import { IssueLoader, HTMLTable, ChannelsParser } from '../../core'
|
||||
import { Logger, Storage, Collection } from '@freearhey/core'
|
||||
import { ChannelList, Issue, Site } from '../../models'
|
||||
import { SITES_DIR, ROOT_DIR } from '../../constants'
|
||||
import { Channel } from 'epg-grabber'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger({ level: -999 })
|
||||
const issueLoader = new IssueLoader()
|
||||
const sitesStorage = new Storage(SITES_DIR)
|
||||
const sites = new Collection()
|
||||
|
||||
logger.info('loading channels...')
|
||||
const channelsParser = new ChannelsParser({
|
||||
storage: sitesStorage
|
||||
})
|
||||
|
||||
logger.info('loading list of sites')
|
||||
const folders = await sitesStorage.list('*/')
|
||||
|
||||
logger.info('loading issues...')
|
||||
const issues = await issueLoader.load()
|
||||
|
||||
logger.info('putting the data together...')
|
||||
const brokenGuideReports = issues.filter(issue =>
|
||||
issue.labels.find((label: string) => label === 'broken guide')
|
||||
)
|
||||
for (const domain of folders) {
|
||||
const filteredIssues = brokenGuideReports.filter(
|
||||
(issue: Issue) => domain === issue.data.get('site')
|
||||
)
|
||||
|
||||
const site = new Site({
|
||||
domain,
|
||||
issues: filteredIssues
|
||||
})
|
||||
|
||||
const files = await sitesStorage.list(`${domain}/*.channels.xml`)
|
||||
for (const filepath of files) {
|
||||
const channelList: ChannelList = await channelsParser.parse(filepath)
|
||||
|
||||
site.totalChannels += channelList.channels.count()
|
||||
site.markedChannels += channelList.channels
|
||||
.filter((channel: Channel) => channel.xmltv_id)
|
||||
.count()
|
||||
}
|
||||
|
||||
sites.add(site)
|
||||
}
|
||||
|
||||
logger.info('creating sites table...')
|
||||
const tableData = new Collection()
|
||||
sites.forEach((site: Site) => {
|
||||
tableData.add([
|
||||
{ value: `<a href="sites/${site.domain}">${site.domain}</a>` },
|
||||
{ value: site.totalChannels, align: 'right' },
|
||||
{ value: site.markedChannels, align: 'right' },
|
||||
{ value: site.getStatus().emoji, align: 'center' },
|
||||
{ value: site.getIssues().all().join(', ') }
|
||||
])
|
||||
})
|
||||
|
||||
logger.info('updating sites.md...')
|
||||
const table = new HTMLTable(tableData.all(), [
|
||||
{ name: 'Site', align: 'left' },
|
||||
{ name: 'Channels<br>(total / with xmltv-id)', colspan: 2, align: 'left' },
|
||||
{ name: 'Status', align: 'left' },
|
||||
{ name: 'Notes', align: 'left' }
|
||||
])
|
||||
const rootStorage = new Storage(ROOT_DIR)
|
||||
const sitesTemplate = await new Storage().load('scripts/templates/_sites.md')
|
||||
const sitesContent = sitesTemplate.replace('_TABLE_', table.toString())
|
||||
await rootStorage.save('SITES.md', sitesContent)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user