diff --git a/scripts/commands/playlist/update.ts b/scripts/commands/playlist/update.ts index 5fb8c23a79..5436443cf6 100644 --- a/scripts/commands/playlist/update.ts +++ b/scripts/commands/playlist/update.ts @@ -1,4 +1,4 @@ -import { isURI, getStreamInfo, loadIssues } from '../../utils' +import { isURI, getStreamInfo, loadIssues, createThread } from '../../utils' import { STREAMS_DIR, LOGS_DIR } from '../../constants' import { Playlist, Issue, Stream } from '../../models' import { loadData, data as apiData } from '../../api' @@ -7,11 +7,13 @@ import { Storage } from '@freearhey/storage-js' import { PlaylistParser } from '../../core' import * as sdk from '@iptv-org/sdk' -const processedIssues = new Collection() +const processedIssues = new Collection() +const skippedIssues = new Collection() +const logger = new Logger({ level: 5 }) + +let streams = new Collection() async function main() { - const logger = new Logger({ level: -999 }) - logger.info('loading data from api...') await loadData() @@ -19,22 +21,27 @@ async function main() { const issues = await loadIssues() logger.info('loading streams...') - const streams = await loadStreams() + await loadStreams() - logger.info('removing streams...') - await removeStreams({ streams, issues }) - - logger.info('edit stream description...') - await editStreams({ streams, issues }) - - logger.info('add new streams...') - await addStreams({ streams, issues }) + logger.info('processing issues...') + await processIssues(issues) logger.info('saving streams...') - await saveStreams({ streams }) + await saveStreams() logger.info('saving logs...') await saveLogs() + + logger.info( + `skipped ${skippedIssues.count()} issue(s): ${skippedIssues + .map((issue: Issue) => `#${issue.number}`) + .join(', ')}` + ) + logger.info( + `processed ${processedIssues.count()} issue(s): ${processedIssues + .map((issue: Issue) => `#${issue.number}`) + .join(', ')}` + ) } main() @@ -45,7 +52,7 @@ async function saveLogs() { await logStorage.save('playlist_update.log', output) } -async function saveStreams({ streams }) { +async function saveStreams() { const streamsStorage = new Storage(STREAMS_DIR) const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath()) for (const filepath of groupedStreams.keys()) { @@ -64,133 +71,118 @@ async function loadStreams() { }) const files = await streamsStorage.list('**/*.m3u') - return await parser.parse(files) + streams = await parser.parse(files) } -async function removeStreams({ - streams, - issues -}: { - streams: Collection - issues: Collection -}) { - const requests = issues.filter( - issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved') - ) +async function processIssues(issues: Collection) { + const requests = issues.filter((issue: Issue) => issue.labels.includes('approved')).all() - requests.forEach((issue: Issue) => { - const data = issue.data - if (data.missing('stream_url')) return - - const streamUrls = data.getString('stream_url') || '' - - let changed = false - streamUrls - .split(/\r?\n/) - .filter(Boolean) - .forEach(link => { - const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim()) - if (found) { - found.removed = true - changed = true - } - }) - - if (changed) processedIssues.add(issue) - }) -} - -async function editStreams({ - streams, - issues -}: { - streams: Collection - issues: Collection -}) { - const requests = issues.filter( - issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved') - ) - requests.forEach((issue: Issue) => { - const data = issue.data - - if (data.missing('stream_url')) return - - const stream: Stream = streams.first( - (_stream: Stream) => _stream.url === data.getString('stream_url') - ) - if (!stream) return - - const streamId = data.getString('stream_id') || '' - const [channelId, feedId] = streamId.split('@') - - if (channelId) { - stream.channel = channelId - stream.feed = feedId - stream.updateTvgId().updateTitle().updateFilepath() + for (const issue of requests) { + switch (true) { + case issue.labels.includes('streams:remove'): + await removeStream(issue) + break + case issue.labels.includes('streams:edit'): + await editStream(issue) + break + case issue.labels.includes('streams:add'): + await addStream(issue) + break } - - stream.updateWithIssue(data) - - processedIssues.add(issue) - }) -} - -async function addStreams({ - streams, - issues -}: { - streams: Collection - issues: Collection -}) { - const requests = issues.filter( - issue => issue.labels.includes('streams:add') && issue.labels.includes('approved') - ) - - for (const issue of requests.all()) { - const data = issue.data - if (data.missing('stream_id') || data.missing('stream_url')) continue - if (streams.includes((_stream: Stream) => _stream.url === data.getString('stream_url'))) - continue - const streamUrl = data.getString('stream_url') || '' - if (!isURI(streamUrl)) continue - - const streamId = data.getString('stream_id') || '' - const [channelId, feedId] = streamId.split('@') - - const channel: sdk.Models.Channel | undefined = apiData.channelsKeyById.get(channelId) - if (!channel) continue - - const label = data.getString('label') || '' - const httpUserAgent = data.getString('http_user_agent') || null - const httpReferrer = data.getString('http_referrer') || null - - let quality = data.getString('quality') || null - if (!quality) { - const streamInfo = await getStreamInfo(streamUrl, { httpUserAgent, httpReferrer }) - - if (streamInfo) { - const height = streamInfo?.resolution?.height - - if (height) { - quality = `${height}p` - } - } - } - - const stream = new Stream({ - channel: channelId, - feed: feedId, - title: channel.name, - url: streamUrl, - user_agent: httpUserAgent, - referrer: httpReferrer, - quality - }) - - stream.label = label - stream.updateTitle().updateFilepath() - - streams.add(stream) - processedIssues.add(issue) } } + +async function removeStream(issue: Issue) { + const log = createThread(issue, 'streams/remove') + log.start() + + const data = issue.data + if (data.missing('stream_url')) return + + const streamUrls = data.getString('stream_url') || '' + + let changed = false + streamUrls + .split(/\r?\n/) + .filter(Boolean) + .forEach(link => { + const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim()) + if (found) { + found.removed = true + changed = true + } + }) + + if (changed) processedIssues.add(issue) +} + +async function editStream(issue: Issue) { + const data = issue.data + + if (data.missing('stream_url')) return + + const stream: Stream = streams.first( + (_stream: Stream) => _stream.url === data.getString('stream_url') + ) + if (!stream) return + + const streamId = data.getString('stream_id') || '' + const [channelId, feedId] = streamId.split('@') + + if (channelId) { + stream.channel = channelId + stream.feed = feedId + stream.updateTvgId().updateTitle().updateFilepath() + } + + stream.updateWithIssue(data) + + processedIssues.add(issue) +} + +async function addStream(issue: Issue) { + const data = issue.data + if (data.missing('stream_id') || data.missing('stream_url')) return + if (streams.includes((_stream: Stream) => _stream.url === data.getString('stream_url'))) return + const streamUrl = data.getString('stream_url') || '' + if (!isURI(streamUrl)) return + + const streamId = data.getString('stream_id') || '' + const [channelId, feedId] = streamId.split('@') + + const channel: sdk.Models.Channel | undefined = apiData.channelsKeyById.get(channelId) + if (!channel) return + + const label = data.getString('label') || '' + const httpUserAgent = data.getString('http_user_agent') || null + const httpReferrer = data.getString('http_referrer') || null + + let quality = data.getString('quality') || null + if (!quality) { + const streamInfo = await getStreamInfo(streamUrl, { httpUserAgent, httpReferrer }) + + if (streamInfo) { + const height = streamInfo?.resolution?.height + + if (height) { + quality = `${height}p` + } + } + } + + const stream = new Stream({ + channel: channelId, + feed: feedId, + title: channel.name, + url: streamUrl, + user_agent: httpUserAgent, + referrer: httpReferrer, + quality + }) + + stream.label = label + stream.updateTitle().updateFilepath() + + streams.add(stream) + processedIssues.add(issue) +} diff --git a/scripts/utils.ts b/scripts/utils.ts index 439a143363..929a379ebb 100644 --- a/scripts/utils.ts +++ b/scripts/utils.ts @@ -167,6 +167,7 @@ export async function loadIssues(props?: { labels: string | string[] }) { per_page: 100, labels, status: 'open', + direction: 'asc', headers: { 'X-GitHub-Api-Version': '2022-11-28' } @@ -293,3 +294,33 @@ function parseDiscussion(discussion: { data: new DataSet(data) }) } + +class LogThread { + issue: Issue + type: string + + constructor(issue: Issue, type: string) { + this.issue = issue + this.type = type + } + + start() { + console.log(`[#${this.issue.number}] ${this.type}: Issue #${this.issue.number}`) + } + + warn(message: string) { + console.log(`[#${this.issue.number}] ${this.type}: └── WARNING: ${message}`) + } + + error(message: string) { + console.log(`[#${this.issue.number}] ${this.type}: └── ERROR: ${message}`) + } + + info(message: string) { + console.log(`[#${this.issue.number}] ${this.type}: └── INFO: ${message}`) + } +} + +export function createThread(issue: Issue, type: string): LogThread { + return new LogThread(issue, type) +} diff --git a/tests/__data__/expected/playlist_update/playlist_update.log b/tests/__data__/expected/playlist_update/playlist_update.log index a15e8a49d8..e715c918c2 100644 --- a/tests/__data__/expected/playlist_update/playlist_update.log +++ b/tests/__data__/expected/playlist_update/playlist_update.log @@ -1 +1 @@ -closes #14151, closes #14150, closes #14110, closes #14120, closes #14175, closes #14105, closes #14104, closes #14057, closes #14034, closes #13964, closes #13893, closes #13881, closes #13793, closes #13751, closes #13715 \ No newline at end of file +closes #14175, closes #14105, closes #14104, closes #14057, closes #14034, closes #13964, closes #13893, closes #13881, closes #13793, closes #13751, closes #13715, closes #14110, closes #14120, closes #14151, closes #14150 \ No newline at end of file