mirror of
https://github.com/iptv-org/epg
synced 2026-04-25 20:17:01 -04:00
Update scripts
This commit is contained in:
freearhey
@@ -1,16 +0,0 @@
|
||||
import axios, { AxiosInstance, AxiosResponse, AxiosRequestConfig } from 'axios'
|
||||
|
||||
export class ApiClient {
|
||||
instance: AxiosInstance
|
||||
|
||||
constructor() {
|
||||
this.instance = axios.create({
|
||||
baseURL: 'https://iptv-org.github.io/api',
|
||||
responseType: 'stream'
|
||||
})
|
||||
}
|
||||
|
||||
get(url: string, options: AxiosRequestConfig): Promise<AxiosResponse> {
|
||||
return this.instance.get(url, options)
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
import { parseChannels } from 'epg-grabber'
|
||||
import { Storage } from '@freearhey/core'
|
||||
import { ChannelList } from '../models'
|
||||
|
||||
interface ChannelsParserProps {
|
||||
storage: Storage
|
||||
}
|
||||
|
||||
export class ChannelsParser {
|
||||
storage: Storage
|
||||
|
||||
constructor({ storage }: ChannelsParserProps) {
|
||||
this.storage = storage
|
||||
}
|
||||
|
||||
async parse(filepath: string): Promise<ChannelList> {
|
||||
const content = await this.storage.load(filepath)
|
||||
const parsed = parseChannels(content)
|
||||
|
||||
return new ChannelList({ channels: parsed })
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { SiteConfig } from 'epg-grabber'
|
||||
import { pathToFileURL } from 'url'
|
||||
|
||||
export class ConfigLoader {
|
||||
async load(filepath: string): Promise<SiteConfig> {
|
||||
const fileUrl = pathToFileURL(filepath).toString()
|
||||
const config = (await import(fileUrl)).default
|
||||
const defaultConfig = {
|
||||
days: 1,
|
||||
delay: 0,
|
||||
output: 'guide.xml',
|
||||
request: {
|
||||
method: 'GET',
|
||||
maxContentLength: 5242880,
|
||||
timeout: 30000,
|
||||
withCredentials: true,
|
||||
jar: null,
|
||||
responseType: 'arraybuffer',
|
||||
cache: false,
|
||||
headers: null,
|
||||
data: null
|
||||
},
|
||||
maxConnections: 1,
|
||||
site: undefined,
|
||||
url: undefined,
|
||||
parser: undefined,
|
||||
channels: undefined
|
||||
}
|
||||
|
||||
return { ...defaultConfig, ...config } as SiteConfig
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
import type { DataLoaderProps, DataLoaderData } from '../types/dataLoader'
|
||||
import cliProgress, { MultiBar } from 'cli-progress'
|
||||
import { Storage } from '@freearhey/core'
|
||||
import { ApiClient } from './apiClient'
|
||||
import numeral from 'numeral'
|
||||
|
||||
export class DataLoader {
|
||||
client: ApiClient
|
||||
storage: Storage
|
||||
progressBar: MultiBar
|
||||
|
||||
constructor(props: DataLoaderProps) {
|
||||
this.client = new ApiClient()
|
||||
this.storage = props.storage
|
||||
this.progressBar = new cliProgress.MultiBar({
|
||||
stopOnComplete: true,
|
||||
hideCursor: true,
|
||||
forceRedraw: true,
|
||||
barsize: 36,
|
||||
format(options, params, payload) {
|
||||
const filename = payload.filename.padEnd(18, ' ')
|
||||
const barsize = options.barsize || 40
|
||||
const percent = (params.progress * 100).toFixed(2)
|
||||
const speed = payload.speed ? numeral(payload.speed).format('0.0 b') + '/s' : 'N/A'
|
||||
const total = numeral(params.total).format('0.0 b')
|
||||
const completeSize = Math.round(params.progress * barsize)
|
||||
const incompleteSize = barsize - completeSize
|
||||
const bar =
|
||||
options.barCompleteString && options.barIncompleteString
|
||||
? options.barCompleteString.substr(0, completeSize) +
|
||||
options.barGlue +
|
||||
options.barIncompleteString.substr(0, incompleteSize)
|
||||
: '-'.repeat(barsize)
|
||||
|
||||
return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async load(): Promise<DataLoaderData> {
|
||||
const [
|
||||
countries,
|
||||
regions,
|
||||
subdivisions,
|
||||
languages,
|
||||
categories,
|
||||
blocklist,
|
||||
channels,
|
||||
feeds,
|
||||
timezones,
|
||||
guides,
|
||||
streams,
|
||||
logos
|
||||
] = await Promise.all([
|
||||
this.storage.json('countries.json'),
|
||||
this.storage.json('regions.json'),
|
||||
this.storage.json('subdivisions.json'),
|
||||
this.storage.json('languages.json'),
|
||||
this.storage.json('categories.json'),
|
||||
this.storage.json('blocklist.json'),
|
||||
this.storage.json('channels.json'),
|
||||
this.storage.json('feeds.json'),
|
||||
this.storage.json('timezones.json'),
|
||||
this.storage.json('guides.json'),
|
||||
this.storage.json('streams.json'),
|
||||
this.storage.json('logos.json')
|
||||
])
|
||||
|
||||
return {
|
||||
countries,
|
||||
regions,
|
||||
subdivisions,
|
||||
languages,
|
||||
categories,
|
||||
blocklist,
|
||||
channels,
|
||||
feeds,
|
||||
timezones,
|
||||
guides,
|
||||
streams,
|
||||
logos
|
||||
}
|
||||
}
|
||||
|
||||
async download(filename: string) {
|
||||
if (!this.storage || !this.progressBar) return
|
||||
|
||||
const stream = await this.storage.createStream(filename)
|
||||
const progressBar = this.progressBar.create(0, 0, { filename })
|
||||
|
||||
this.client
|
||||
.get(filename, {
|
||||
responseType: 'stream',
|
||||
onDownloadProgress({ total, loaded, rate }) {
|
||||
if (total) progressBar.setTotal(total)
|
||||
progressBar.update(loaded, { speed: rate })
|
||||
}
|
||||
})
|
||||
.then(response => {
|
||||
response.data.pipe(stream)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
import { Channel, Feed, GuideChannel, Logo, Stream } from '../models'
|
||||
import { DataLoaderData } from '../types/dataLoader'
|
||||
import { Collection } from '@freearhey/core'
|
||||
|
||||
export class DataProcessor {
|
||||
|
||||
process(data: DataLoaderData) {
|
||||
let channels = new Collection(data.channels).map(data => new Channel(data))
|
||||
const channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
|
||||
|
||||
const guideChannels = new Collection(data.guides).map(data => new GuideChannel(data))
|
||||
const guideChannelsGroupedByStreamId = guideChannels.groupBy((channel: GuideChannel) =>
|
||||
channel.getStreamId()
|
||||
)
|
||||
|
||||
const streams = new Collection(data.streams).map(data => new Stream(data))
|
||||
const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
|
||||
|
||||
let feeds = new Collection(data.feeds).map(data =>
|
||||
new Feed(data)
|
||||
.withGuideChannels(guideChannelsGroupedByStreamId)
|
||||
.withStreams(streamsGroupedById)
|
||||
.withChannel(channelsKeyById)
|
||||
)
|
||||
const feedsKeyByStreamId = feeds.keyBy((feed: Feed) => feed.getStreamId())
|
||||
|
||||
const logos = new Collection(data.logos).map(data =>
|
||||
new Logo(data).withFeed(feedsKeyByStreamId)
|
||||
)
|
||||
const logosGroupedByChannelId = logos.groupBy((logo: Logo) => logo.channelId)
|
||||
const logosGroupedByStreamId = logos.groupBy((logo: Logo) => logo.getStreamId())
|
||||
|
||||
feeds = feeds.map((feed: Feed) => feed.withLogos(logosGroupedByStreamId))
|
||||
const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
|
||||
|
||||
channels = channels.map((channel: Channel) =>
|
||||
channel.withFeeds(feedsGroupedByChannelId).withLogos(logosGroupedByChannelId)
|
||||
)
|
||||
|
||||
return {
|
||||
guideChannelsGroupedByStreamId,
|
||||
feedsGroupedByChannelId,
|
||||
logosGroupedByChannelId,
|
||||
logosGroupedByStreamId,
|
||||
streamsGroupedById,
|
||||
feedsKeyByStreamId,
|
||||
channelsKeyById,
|
||||
guideChannels,
|
||||
channels,
|
||||
streams,
|
||||
feeds,
|
||||
logos
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
import dayjs from 'dayjs'
|
||||
import utc from 'dayjs/plugin/utc'
|
||||
|
||||
dayjs.extend(utc)
|
||||
|
||||
const date = {}
|
||||
|
||||
date.getUTC = function (d = null) {
|
||||
if (typeof d === 'string') return dayjs.utc(d).startOf('d')
|
||||
|
||||
return dayjs.utc().startOf('d')
|
||||
}
|
||||
|
||||
export default date
|
||||
@@ -1,105 +0,0 @@
|
||||
import { EPGGrabber, GrabCallbackData, EPGGrabberMock, SiteConfig, Channel } from 'epg-grabber'
|
||||
import { Logger, Collection } from '@freearhey/core'
|
||||
import { Queue, ProxyParser } from './'
|
||||
import { GrabOptions } from '../commands/epg/grab'
|
||||
import { TaskQueue, PromisyClass } from 'cwait'
|
||||
import { SocksProxyAgent } from 'socks-proxy-agent'
|
||||
|
||||
interface GrabberProps {
|
||||
logger: Logger
|
||||
queue: Queue
|
||||
options: GrabOptions
|
||||
}
|
||||
|
||||
export class Grabber {
|
||||
logger: Logger
|
||||
queue: Queue
|
||||
options: GrabOptions
|
||||
grabber: EPGGrabber | EPGGrabberMock
|
||||
|
||||
constructor({ logger, queue, options }: GrabberProps) {
|
||||
this.logger = logger
|
||||
this.queue = queue
|
||||
this.options = options
|
||||
this.grabber = process.env.NODE_ENV === 'test' ? new EPGGrabberMock() : new EPGGrabber()
|
||||
}
|
||||
|
||||
async grab(): Promise<{ channels: Collection; programs: Collection }> {
|
||||
const proxyParser = new ProxyParser()
|
||||
const taskQueue = new TaskQueue(Promise as PromisyClass, this.options.maxConnections)
|
||||
|
||||
const total = this.queue.size()
|
||||
|
||||
const channels = new Collection()
|
||||
let programs = new Collection()
|
||||
let i = 1
|
||||
|
||||
await Promise.all(
|
||||
this.queue.items().map(
|
||||
taskQueue.wrap(
|
||||
async (queueItem: { channel: Channel; config: SiteConfig; date: string }) => {
|
||||
const { channel, config, date } = queueItem
|
||||
|
||||
channels.add(channel)
|
||||
|
||||
if (this.options.timeout !== undefined) {
|
||||
const timeout = parseInt(this.options.timeout)
|
||||
config.request = { ...config.request, ...{ timeout } }
|
||||
}
|
||||
|
||||
if (this.options.delay !== undefined) {
|
||||
const delay = parseInt(this.options.delay)
|
||||
config.delay = delay
|
||||
}
|
||||
|
||||
if (this.options.proxy !== undefined) {
|
||||
const proxy = proxyParser.parse(this.options.proxy)
|
||||
|
||||
if (
|
||||
proxy.protocol &&
|
||||
['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol))
|
||||
) {
|
||||
const socksProxyAgent = new SocksProxyAgent(this.options.proxy)
|
||||
|
||||
config.request = {
|
||||
...config.request,
|
||||
...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent }
|
||||
}
|
||||
} else {
|
||||
config.request = { ...config.request, ...{ proxy } }
|
||||
}
|
||||
}
|
||||
|
||||
if (this.options.curl === true) {
|
||||
config.curl = true
|
||||
}
|
||||
|
||||
const _programs = await this.grabber.grab(
|
||||
channel,
|
||||
date,
|
||||
config,
|
||||
(data: GrabCallbackData, error: Error | null) => {
|
||||
const { programs, date } = data
|
||||
|
||||
this.logger.info(
|
||||
` [${i}/${total}] ${channel.site} (${channel.lang}) - ${
|
||||
channel.xmltv_id
|
||||
} - ${date.format('MMM D, YYYY')} (${programs.length} programs)`
|
||||
)
|
||||
if (i < total) i++
|
||||
|
||||
if (error) {
|
||||
this.logger.info(` ERR: ${error.message}`)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
programs = programs.concat(new Collection(_programs))
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return { channels, programs }
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
import { Collection, Logger, Zip, Storage, StringTemplate } from '@freearhey/core'
|
||||
import epgGrabber from 'epg-grabber'
|
||||
import { OptionValues } from 'commander'
|
||||
import { Channel, Feed, Guide } from '../models'
|
||||
import path from 'path'
|
||||
import { DataLoader, DataProcessor } from '.'
|
||||
import { DataLoaderData } from '../types/dataLoader'
|
||||
import { DataProcessorData } from '../types/dataProcessor'
|
||||
import { DATA_DIR } from '../constants'
|
||||
|
||||
interface GuideManagerProps {
|
||||
options: OptionValues
|
||||
logger: Logger
|
||||
channels: Collection
|
||||
programs: Collection
|
||||
}
|
||||
|
||||
export class GuideManager {
|
||||
options: OptionValues
|
||||
logger: Logger
|
||||
channels: Collection
|
||||
programs: Collection
|
||||
|
||||
constructor({ channels, programs, logger, options }: GuideManagerProps) {
|
||||
this.options = options
|
||||
this.logger = logger
|
||||
this.channels = channels
|
||||
this.programs = programs
|
||||
}
|
||||
|
||||
async createGuides() {
|
||||
const pathTemplate = new StringTemplate(this.options.output)
|
||||
|
||||
const processor = new DataProcessor()
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const loader = new DataLoader({ storage: dataStorage })
|
||||
const data: DataLoaderData = await loader.load()
|
||||
const { feedsKeyByStreamId, channelsKeyById }: DataProcessorData = processor.process(data)
|
||||
|
||||
const groupedChannels = this.channels
|
||||
.map((channel: epgGrabber.Channel) => {
|
||||
if (channel.xmltv_id && !channel.icon) {
|
||||
const foundFeed: Feed = feedsKeyByStreamId.get(channel.xmltv_id)
|
||||
if (foundFeed && foundFeed.hasLogo()) {
|
||||
channel.icon = foundFeed.getLogoUrl()
|
||||
} else {
|
||||
const [channelId] = channel.xmltv_id.split('@')
|
||||
const foundChannel: Channel = channelsKeyById.get(channelId)
|
||||
if (foundChannel && foundChannel.hasLogo()) {
|
||||
channel.icon = foundChannel.getLogoUrl()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return channel
|
||||
})
|
||||
.orderBy([
|
||||
(channel: epgGrabber.Channel) => channel.index,
|
||||
(channel: epgGrabber.Channel) => channel.xmltv_id
|
||||
])
|
||||
.uniqBy(
|
||||
(channel: epgGrabber.Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`
|
||||
)
|
||||
.groupBy((channel: epgGrabber.Channel) => {
|
||||
return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' })
|
||||
})
|
||||
|
||||
const groupedPrograms = this.programs
|
||||
.orderBy([
|
||||
(program: epgGrabber.Program) => program.channel,
|
||||
(program: epgGrabber.Program) => program.start
|
||||
])
|
||||
.groupBy((program: epgGrabber.Program) => {
|
||||
const lang =
|
||||
program.titles && program.titles.length && program.titles[0].lang
|
||||
? program.titles[0].lang
|
||||
: 'en'
|
||||
|
||||
return pathTemplate.format({ lang, site: program.site || '' })
|
||||
})
|
||||
|
||||
for (const groupKey of groupedPrograms.keys()) {
|
||||
const guide = new Guide({
|
||||
filepath: groupKey,
|
||||
gzip: this.options.gzip,
|
||||
channels: new Collection(groupedChannels.get(groupKey)),
|
||||
programs: new Collection(groupedPrograms.get(groupKey))
|
||||
})
|
||||
|
||||
await this.save(guide)
|
||||
}
|
||||
}
|
||||
|
||||
async save(guide: Guide) {
|
||||
const storage = new Storage(path.dirname(guide.filepath))
|
||||
const xmlFilepath = guide.filepath
|
||||
const xmlFilename = path.basename(xmlFilepath)
|
||||
this.logger.info(` saving to "${xmlFilepath}"...`)
|
||||
const xmltv = guide.toString()
|
||||
await storage.save(xmlFilename, xmltv)
|
||||
|
||||
if (guide.gzip) {
|
||||
const zip = new Zip()
|
||||
const compressed = zip.compress(xmltv)
|
||||
const gzFilepath = `${guide.filepath}.gz`
|
||||
const gzFilename = path.basename(gzFilepath)
|
||||
this.logger.info(` saving to "${gzFilepath}"...`)
|
||||
await storage.save(gzFilename, compressed)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,55 +1,45 @@
|
||||
interface Column {
|
||||
name: string
|
||||
nowrap?: boolean
|
||||
align?: string
|
||||
colspan?: number
|
||||
}
|
||||
|
||||
type DataItem = {
|
||||
value: string
|
||||
nowrap?: boolean
|
||||
align?: string
|
||||
colspan?: number
|
||||
}[]
|
||||
|
||||
export class HTMLTable {
|
||||
data: DataItem[]
|
||||
columns: Column[]
|
||||
|
||||
constructor(data: DataItem[], columns: Column[]) {
|
||||
this.data = data
|
||||
this.columns = columns
|
||||
}
|
||||
|
||||
toString() {
|
||||
let output = '<table>\r\n'
|
||||
|
||||
output += ' <thead>\r\n <tr>'
|
||||
for (const column of this.columns) {
|
||||
const nowrap = column.nowrap ? ' nowrap' : ''
|
||||
const align = column.align ? ` align="${column.align}"` : ''
|
||||
const colspan = column.colspan ? ` colspan="${column.colspan}"` : ''
|
||||
|
||||
output += `<th${align}${nowrap}${colspan}>${column.name}</th>`
|
||||
}
|
||||
output += '</tr>\r\n </thead>\r\n'
|
||||
|
||||
output += ' <tbody>\r\n'
|
||||
for (const row of this.data) {
|
||||
output += ' <tr>'
|
||||
for (const item of row) {
|
||||
const nowrap = item.nowrap ? ' nowrap' : ''
|
||||
const align = item.align ? ` align="${item.align}"` : ''
|
||||
const colspan = item.colspan ? ` colspan="${item.colspan}"` : ''
|
||||
|
||||
output += `<td${align}${nowrap}${colspan}>${item.value}</td>`
|
||||
}
|
||||
output += '</tr>\r\n'
|
||||
}
|
||||
output += ' </tbody>\r\n'
|
||||
|
||||
output += '</table>'
|
||||
|
||||
return output
|
||||
}
|
||||
}
|
||||
import { HTMLTableColumn, HTMLTableDataItem, HTMLTableRow } from '../types/htmlTable'
|
||||
import { Collection } from '@freearhey/core'
|
||||
import { EOL } from '../constants'
|
||||
|
||||
export class HTMLTable {
|
||||
rows: Collection<HTMLTableRow>
|
||||
columns: Collection<HTMLTableColumn>
|
||||
|
||||
constructor(rows: Collection<HTMLTableRow>, columns: Collection<HTMLTableColumn>) {
|
||||
this.rows = rows
|
||||
this.columns = columns
|
||||
}
|
||||
|
||||
toString() {
|
||||
let output = `<table>${EOL}`
|
||||
|
||||
output += ` <thead>${EOL} <tr>`
|
||||
this.columns.forEach((column: HTMLTableColumn) => {
|
||||
const nowrap = column.nowrap ? ' nowrap' : ''
|
||||
const align = column.align ? ` align="${column.align}"` : ''
|
||||
const colspan = column.colspan ? ` colspan="${column.colspan}"` : ''
|
||||
|
||||
output += `<th${align}${nowrap}${colspan}>${column.name}</th>`
|
||||
})
|
||||
output += `</tr>${EOL} </thead>${EOL}`
|
||||
|
||||
output += ` <tbody>${EOL}`
|
||||
this.rows.forEach((row: HTMLTableRow) => {
|
||||
output += ' <tr>'
|
||||
row.forEach((item: HTMLTableDataItem) => {
|
||||
const nowrap = item.nowrap ? ' nowrap' : ''
|
||||
const align = item.align ? ` align="${item.align}"` : ''
|
||||
const colspan = item.colspan ? ` colspan="${item.colspan}"` : ''
|
||||
|
||||
output += `<td${align}${nowrap}${colspan}>${item.value}</td>`
|
||||
})
|
||||
output += `</tr>${EOL}`
|
||||
})
|
||||
output += ` </tbody>${EOL}`
|
||||
|
||||
output += '</table>'
|
||||
|
||||
return output
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,4 @@
|
||||
export * from './apiClient'
|
||||
export * from './channelsParser'
|
||||
export * from './configLoader'
|
||||
export * from './dataLoader'
|
||||
export * from './dataProcessor'
|
||||
export * from './grabber'
|
||||
export * from './guideManager'
|
||||
export * from './htmlTable'
|
||||
export * from './issueLoader'
|
||||
export * from './issueParser'
|
||||
export * from './job'
|
||||
export * from './proxyParser'
|
||||
export * from './queue'
|
||||
export * from './queueCreator'
|
||||
export * from './htmlTable'
|
||||
export * from './siteConfig'
|
||||
export * from './utils'
|
||||
export * from './queue'
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'
|
||||
import { paginateRest } from '@octokit/plugin-paginate-rest'
|
||||
import { TESTING, OWNER, REPO } from '../constants'
|
||||
import { Collection } from '@freearhey/core'
|
||||
import { Octokit } from '@octokit/core'
|
||||
import { IssueParser } from './'
|
||||
|
||||
const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods)
|
||||
const octokit = new CustomOctokit()
|
||||
|
||||
export class IssueLoader {
|
||||
async load(props?: { labels: string[] | string }) {
|
||||
let labels = ''
|
||||
if (props && props.labels) {
|
||||
labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels
|
||||
}
|
||||
let issues: object[] = []
|
||||
if (TESTING) {
|
||||
issues = (await import('../../tests/__data__/input/sites_update/issues.mjs')).default
|
||||
} else {
|
||||
issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
|
||||
owner: OWNER,
|
||||
repo: REPO,
|
||||
per_page: 100,
|
||||
labels,
|
||||
state: 'open',
|
||||
headers: {
|
||||
'X-GitHub-Api-Version': '2022-11-28'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const parser = new IssueParser()
|
||||
|
||||
return new Collection(issues).map(parser.parse)
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
import { Dictionary } from '@freearhey/core'
|
||||
import { Issue } from '../models'
|
||||
|
||||
const FIELDS = new Dictionary({
|
||||
Site: 'site'
|
||||
})
|
||||
|
||||
export class IssueParser {
|
||||
parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue {
|
||||
const fields = issue.body.split('###')
|
||||
|
||||
const data = new Dictionary()
|
||||
fields.forEach((field: string) => {
|
||||
const parsed = field.split(/\r?\n/).filter(Boolean)
|
||||
let _label = parsed.shift()
|
||||
_label = _label ? _label.trim() : ''
|
||||
let _value = parsed.join('\r\n')
|
||||
_value = _value ? _value.trim() : ''
|
||||
|
||||
if (!_label || !_value) return data
|
||||
|
||||
const id: string = FIELDS.get(_label)
|
||||
const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
|
||||
|
||||
if (!id) return
|
||||
|
||||
data.set(id, value)
|
||||
})
|
||||
|
||||
const labels = issue.labels.map(label => label.name)
|
||||
|
||||
return new Issue({ number: issue.number, labels, data })
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
import { Logger } from '@freearhey/core'
|
||||
import { Queue, Grabber, GuideManager } from '.'
|
||||
import { GrabOptions } from '../commands/epg/grab'
|
||||
|
||||
interface JobProps {
|
||||
options: GrabOptions
|
||||
logger: Logger
|
||||
queue: Queue
|
||||
}
|
||||
|
||||
export class Job {
|
||||
options: GrabOptions
|
||||
logger: Logger
|
||||
grabber: Grabber
|
||||
|
||||
constructor({ queue, logger, options }: JobProps) {
|
||||
this.options = options
|
||||
this.logger = logger
|
||||
this.grabber = new Grabber({ logger, queue, options })
|
||||
}
|
||||
|
||||
async run() {
|
||||
const { channels, programs } = await this.grabber.grab()
|
||||
|
||||
const manager = new GuideManager({
|
||||
channels,
|
||||
programs,
|
||||
options: this.options,
|
||||
logger: this.logger
|
||||
})
|
||||
|
||||
await manager.createGuides()
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
import { URL } from 'node:url'
|
||||
|
||||
interface ProxyParserResult {
|
||||
protocol: string | null
|
||||
auth?: {
|
||||
username?: string
|
||||
password?: string
|
||||
}
|
||||
host: string
|
||||
port: number | null
|
||||
}
|
||||
|
||||
export class ProxyParser {
|
||||
parse(_url: string): ProxyParserResult {
|
||||
const parsed = new URL(_url)
|
||||
|
||||
const result: ProxyParserResult = {
|
||||
protocol: parsed.protocol.replace(':', '') || null,
|
||||
host: parsed.hostname,
|
||||
port: parsed.port ? parseInt(parsed.port) : null
|
||||
}
|
||||
|
||||
if (parsed.username || parsed.password) {
|
||||
result.auth = {}
|
||||
if (parsed.username) result.auth.username = parsed.username
|
||||
if (parsed.password) result.auth.password = parsed.password
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -1,45 +1,18 @@
|
||||
import { Dictionary } from '@freearhey/core'
|
||||
import { SiteConfig, Channel } from 'epg-grabber'
|
||||
|
||||
export interface QueueItem {
|
||||
channel: Channel
|
||||
date: string
|
||||
config: SiteConfig
|
||||
error: string | null
|
||||
}
|
||||
|
||||
export class Queue {
|
||||
_data: Dictionary
|
||||
|
||||
constructor() {
|
||||
this._data = new Dictionary()
|
||||
}
|
||||
|
||||
missing(key: string): boolean {
|
||||
return this._data.missing(key)
|
||||
}
|
||||
|
||||
add(
|
||||
key: string,
|
||||
{ channel, config, date }: { channel: Channel; date: string | null; config: SiteConfig }
|
||||
) {
|
||||
this._data.set(key, {
|
||||
channel,
|
||||
date,
|
||||
config,
|
||||
error: null
|
||||
})
|
||||
}
|
||||
|
||||
size(): number {
|
||||
return Object.values(this._data.data()).length
|
||||
}
|
||||
|
||||
items(): QueueItem[] {
|
||||
return Object.values(this._data.data()) as QueueItem[]
|
||||
}
|
||||
|
||||
isEmpty(): boolean {
|
||||
return this.size() === 0
|
||||
}
|
||||
}
|
||||
import { Collection, Dictionary } from '@freearhey/core'
|
||||
import { QueueItem } from '../types/queue'
|
||||
|
||||
export class Queue {
|
||||
#items: Dictionary<QueueItem> = new Dictionary<QueueItem>()
|
||||
|
||||
add(key: string, data: QueueItem) {
|
||||
this.#items.set(key, data)
|
||||
}
|
||||
|
||||
has(key: string): boolean {
|
||||
return this.#items.has(key)
|
||||
}
|
||||
|
||||
getItems(): Collection<QueueItem> {
|
||||
return new Collection<QueueItem>(Object.values(this.#items.data()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
import { Storage, Collection, DateTime, Logger } from '@freearhey/core'
|
||||
import { SITES_DIR, DATA_DIR } from '../constants'
|
||||
import { GrabOptions } from '../commands/epg/grab'
|
||||
import { ConfigLoader, Queue } from './'
|
||||
import { SiteConfig } from 'epg-grabber'
|
||||
import path from 'path'
|
||||
|
||||
interface QueueCreatorProps {
|
||||
logger: Logger
|
||||
options: GrabOptions
|
||||
channels: Collection
|
||||
}
|
||||
|
||||
export class QueueCreator {
|
||||
configLoader: ConfigLoader
|
||||
logger: Logger
|
||||
sitesStorage: Storage
|
||||
dataStorage: Storage
|
||||
channels: Collection
|
||||
options: GrabOptions
|
||||
|
||||
constructor({ channels, logger, options }: QueueCreatorProps) {
|
||||
this.channels = channels
|
||||
this.logger = logger
|
||||
this.sitesStorage = new Storage()
|
||||
this.dataStorage = new Storage(DATA_DIR)
|
||||
this.options = options
|
||||
this.configLoader = new ConfigLoader()
|
||||
}
|
||||
|
||||
async create(): Promise<Queue> {
|
||||
let index = 0
|
||||
const queue = new Queue()
|
||||
for (const channel of this.channels.all()) {
|
||||
channel.index = index++
|
||||
if (!channel.site || !channel.site_id || !channel.name) continue
|
||||
|
||||
const configPath = path.resolve(SITES_DIR, `${channel.site}/${channel.site}.config.js`)
|
||||
const config: SiteConfig = await this.configLoader.load(configPath)
|
||||
|
||||
if (!channel.xmltv_id) {
|
||||
channel.xmltv_id = channel.site_id
|
||||
}
|
||||
|
||||
const days = this.options.days || config.days || 1
|
||||
const currDate = new DateTime(process.env.CURR_DATE || new Date().toISOString())
|
||||
const dates = Array.from({ length: days }, (_, day) => currDate.add(day, 'd'))
|
||||
dates.forEach((date: DateTime) => {
|
||||
const dateString = date.toJSON()
|
||||
const key = `${channel.site}:${channel.lang}:${channel.xmltv_id}:${dateString}`
|
||||
if (queue.missing(key)) {
|
||||
queue.add(key, {
|
||||
channel,
|
||||
date: dateString,
|
||||
config
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return queue
|
||||
}
|
||||
}
|
||||
71
scripts/core/siteConfig.ts
Normal file
71
scripts/core/siteConfig.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import * as epgGrabber from 'epg-grabber'
|
||||
import _ from 'lodash'
|
||||
|
||||
const _default = {
|
||||
days: 1,
|
||||
delay: 0,
|
||||
output: 'guide.xml',
|
||||
request: {
|
||||
method: 'GET',
|
||||
maxContentLength: 5242880,
|
||||
timeout: 30000,
|
||||
withCredentials: true,
|
||||
jar: null,
|
||||
responseType: 'arraybuffer',
|
||||
cache: false,
|
||||
headers: null,
|
||||
data: null
|
||||
},
|
||||
maxConnections: 1,
|
||||
site: undefined,
|
||||
url: undefined,
|
||||
parser: undefined,
|
||||
channels: undefined,
|
||||
lang: 'en',
|
||||
debug: false,
|
||||
gzip: false,
|
||||
curl: false,
|
||||
logo: ''
|
||||
}
|
||||
|
||||
export class SiteConfig {
|
||||
days: number
|
||||
lang: string
|
||||
delay: number
|
||||
debug: boolean
|
||||
gzip: boolean
|
||||
curl: boolean
|
||||
maxConnections: number
|
||||
output: string
|
||||
request: epgGrabber.Types.SiteConfigRequestConfig
|
||||
site: string
|
||||
channels?: string | string[]
|
||||
url: ((context: epgGrabber.Types.SiteConfigRequestContext) => string | Promise<string>) | string
|
||||
parser: (
|
||||
context: epgGrabber.Types.SiteConfigParserContext
|
||||
) =>
|
||||
| epgGrabber.Types.SiteConfigParserResult[]
|
||||
| Promise<epgGrabber.Types.SiteConfigParserResult[]>
|
||||
logo: ((context: epgGrabber.Types.SiteConfigRequestContext) => string | Promise<string>) | string
|
||||
filepath: string
|
||||
|
||||
constructor(config: epgGrabber.Types.SiteConfigObject) {
|
||||
this.site = config.site
|
||||
this.channels = config.channels
|
||||
this.url = config.url
|
||||
this.parser = config.parser
|
||||
this.filepath = config.filepath
|
||||
|
||||
this.days = config.days || _default.days
|
||||
this.lang = config.lang || _default.lang
|
||||
this.delay = config.delay || _default.delay
|
||||
this.debug = config.debug || _default.debug
|
||||
this.maxConnections = config.maxConnections || _default.maxConnections
|
||||
this.gzip = config.gzip || _default.gzip
|
||||
this.curl = config.curl || _default.curl
|
||||
this.output = config.output || _default.output
|
||||
this.logo = config.logo || _default.logo
|
||||
|
||||
this.request = _.merge(_default.request, config.request)
|
||||
}
|
||||
}
|
||||
106
scripts/core/utils.ts
Normal file
106
scripts/core/utils.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'
|
||||
import { paginateRest } from '@octokit/plugin-paginate-rest'
|
||||
import { TESTING, OWNER, REPO, EOL } from '../constants'
|
||||
import { Collection } from '@freearhey/core'
|
||||
import { Channel } from '../models/channel'
|
||||
import { AxiosProxyConfig } from 'axios'
|
||||
import { Octokit } from '@octokit/core'
|
||||
import { pathToFileURL } from 'url'
|
||||
import { Issue } from '../models'
|
||||
import { URL } from 'node:url'
|
||||
|
||||
export function generateChannelsXML(channels: Collection<Channel>): string {
|
||||
let output = `<?xml version="1.0" encoding="UTF-8"?>${EOL}<channels>${EOL}`
|
||||
|
||||
channels.forEach((channel: Channel) => {
|
||||
const logo = channel.logo ? ` logo="${escapeString(channel.logo)}"` : ''
|
||||
const xmltv_id = channel.xmltv_id ? escapeString(channel.xmltv_id) : ''
|
||||
const lang = channel.lang || ''
|
||||
const site_id = channel.site_id ? escapeString(channel.site_id) : ''
|
||||
const site = channel.site || ''
|
||||
const displayName = channel.name ? escapeString(channel.name) : ''
|
||||
|
||||
output += ` <channel site="${site}" site_id="${site_id}" lang="${lang}"${logo} xmltv_id="${xmltv_id}">${displayName}</channel>${EOL}`
|
||||
})
|
||||
|
||||
output += `</channels>${EOL}`
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
export function escapeString(value: string, defaultValue = '') {
|
||||
if (!value) return defaultValue
|
||||
|
||||
const regex = new RegExp(
|
||||
'((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
|
||||
'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
|
||||
'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
|
||||
'|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
|
||||
'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
|
||||
'[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
|
||||
'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
|
||||
'(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
|
||||
'g'
|
||||
)
|
||||
|
||||
value = String(value || '').replace(regex, '')
|
||||
|
||||
return value
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''')
|
||||
.replace(/\n|\r/g, ' ')
|
||||
.replace(/ +/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
export function parseProxy(string: string): AxiosProxyConfig {
|
||||
const parsed = new URL(string)
|
||||
|
||||
const proxy: AxiosProxyConfig = {
|
||||
protocol: parsed.protocol.replace(':', ''),
|
||||
host: parsed.hostname,
|
||||
port: parsed.port ? parseInt(parsed.port) : 8080
|
||||
}
|
||||
|
||||
if (parsed.username || parsed.password) {
|
||||
proxy.auth = { username: parsed.username, password: parsed.password }
|
||||
}
|
||||
|
||||
return proxy
|
||||
}
|
||||
|
||||
export async function loadJs(filepath: string) {
|
||||
const fileUrl = pathToFileURL(filepath).toString()
|
||||
|
||||
return (await import(fileUrl)).default
|
||||
}
|
||||
|
||||
export async function loadIssues(props?: { labels: string[] | string }) {
|
||||
const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods)
|
||||
const octokit = new CustomOctokit()
|
||||
|
||||
let labels = ''
|
||||
if (props && props.labels) {
|
||||
labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels
|
||||
}
|
||||
let issues: object[] = []
|
||||
if (TESTING) {
|
||||
issues = (await import('../../tests/__data__/input/sites_update/issues.mjs')).default
|
||||
} else {
|
||||
issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
|
||||
owner: OWNER,
|
||||
repo: REPO,
|
||||
per_page: 100,
|
||||
labels,
|
||||
state: 'open',
|
||||
headers: {
|
||||
'X-GitHub-Api-Version': '2022-11-28'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return new Collection(issues).map(data => new Issue(data))
|
||||
}
|
||||
Reference in New Issue
Block a user