wip

scripts/commands/playlist/update.ts

@@ -1,4 +1,4 @@
-import { isURI, getStreamInfo, loadIssues } from '../../utils'
+import { isURI, getStreamInfo, loadIssues, createThread } from '../../utils'
 import { STREAMS_DIR, LOGS_DIR } from '../../constants'
 import { Playlist, Issue, Stream } from '../../models'
 import { loadData, data as apiData } from '../../api'
@@ -7,11 +7,13 @@ import { Storage } from '@freearhey/storage-js'
 import { PlaylistParser } from '../../core'
 import * as sdk from '@iptv-org/sdk'
 
-const processedIssues = new Collection()
+const processedIssues = new Collection<Issue>()
+const skippedIssues = new Collection<Issue>()
+const logger = new Logger({ level: 5 })
+
+let streams = new Collection<Stream>()
 
 async function main() {
-  const logger = new Logger({ level: -999 })
-
   logger.info('loading data from api...')
   await loadData()
 
@@ -19,22 +21,27 @@ async function main() {
   const issues = await loadIssues()
 
   logger.info('loading streams...')
-  const streams = await loadStreams()
+  await loadStreams()
 
-  logger.info('removing streams...')
+  logger.info('processing issues...')
-  await removeStreams({ streams, issues })
+  await processIssues(issues)
-
-  logger.info('edit stream description...')
-  await editStreams({ streams, issues })
-
-  logger.info('add new streams...')
-  await addStreams({ streams, issues })
 
   logger.info('saving streams...')
-  await saveStreams({ streams })
+  await saveStreams()
 
   logger.info('saving logs...')
   await saveLogs()
+
+  logger.info(
+    `skipped ${skippedIssues.count()} issue(s): ${skippedIssues
+      .map((issue: Issue) => `#${issue.number}`)
+      .join(', ')}`
+  )
+  logger.info(
+    `processed ${processedIssues.count()} issue(s): ${processedIssues
+      .map((issue: Issue) => `#${issue.number}`)
+      .join(', ')}`
+  )
 }
 
 main()
@@ -45,7 +52,7 @@ async function saveLogs() {
   await logStorage.save('playlist_update.log', output)
 }
 
-async function saveStreams({ streams }) {
+async function saveStreams() {
   const streamsStorage = new Storage(STREAMS_DIR)
   const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
   for (const filepath of groupedStreams.keys()) {
@@ -64,133 +71,118 @@ async function loadStreams() {
   })
   const files = await streamsStorage.list('**/*.m3u')
 
-  return await parser.parse(files)
+  streams = await parser.parse(files)
 }
 
-async function removeStreams({
+async function processIssues(issues: Collection<Issue>) {
-  streams,
+  const requests = issues.filter((issue: Issue) => issue.labels.includes('approved')).all()
-  issues
-}: {
-  streams: Collection<Stream>
-  issues: Collection<Issue>
-}) {
-  const requests = issues.filter(
-    issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved')
-  )
 
-  requests.forEach((issue: Issue) => {
+  for (const issue of requests) {
-    const data = issue.data
+    switch (true) {
-    if (data.missing('stream_url')) return
+      case issue.labels.includes('streams:remove'):
-
+        await removeStream(issue)
-    const streamUrls = data.getString('stream_url') || ''
+        break
-
+      case issue.labels.includes('streams:edit'):
-    let changed = false
+        await editStream(issue)
-    streamUrls
+        break
-      .split(/\r?\n/)
+      case issue.labels.includes('streams:add'):
-      .filter(Boolean)
+        await addStream(issue)
-      .forEach(link => {
+        break
-        const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim())
-        if (found) {
-          found.removed = true
-          changed = true
-        }
-      })
-
-    if (changed) processedIssues.add(issue)
-  })
-}
-
-async function editStreams({
-  streams,
-  issues
-}: {
-  streams: Collection<Stream>
-  issues: Collection<Issue>
-}) {
-  const requests = issues.filter(
-    issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved')
-  )
-  requests.forEach((issue: Issue) => {
-    const data = issue.data
-
-    if (data.missing('stream_url')) return
-
-    const stream: Stream = streams.first(
-      (_stream: Stream) => _stream.url === data.getString('stream_url')
-    )
-    if (!stream) return
-
-    const streamId = data.getString('stream_id') || ''
-    const [channelId, feedId] = streamId.split('@')
-
-    if (channelId) {
-      stream.channel = channelId
-      stream.feed = feedId
-      stream.updateTvgId().updateTitle().updateFilepath()
     }
-
-    stream.updateWithIssue(data)
-
-    processedIssues.add(issue)
-  })
-}
-
-async function addStreams({
-  streams,
-  issues
-}: {
-  streams: Collection<Stream>
-  issues: Collection<Issue>
-}) {
-  const requests = issues.filter(
-    issue => issue.labels.includes('streams:add') && issue.labels.includes('approved')
-  )
-
-  for (const issue of requests.all()) {
-    const data = issue.data
-    if (data.missing('stream_id') || data.missing('stream_url')) continue
-    if (streams.includes((_stream: Stream) => _stream.url === data.getString('stream_url')))
-      continue
-    const streamUrl = data.getString('stream_url') || ''
-    if (!isURI(streamUrl)) continue
-
-    const streamId = data.getString('stream_id') || ''
-    const [channelId, feedId] = streamId.split('@')
-
-    const channel: sdk.Models.Channel | undefined = apiData.channelsKeyById.get(channelId)
-    if (!channel) continue
-
-    const label = data.getString('label') || ''
-    const httpUserAgent = data.getString('http_user_agent') || null
-    const httpReferrer = data.getString('http_referrer') || null
-
-    let quality = data.getString('quality') || null
-    if (!quality) {
-      const streamInfo = await getStreamInfo(streamUrl, { httpUserAgent, httpReferrer })
-
-      if (streamInfo) {
-        const height = streamInfo?.resolution?.height
-
-        if (height) {
-          quality = `${height}p`
-        }
-      }
-    }
-
-    const stream = new Stream({
-      channel: channelId,
-      feed: feedId,
-      title: channel.name,
-      url: streamUrl,
-      user_agent: httpUserAgent,
-      referrer: httpReferrer,
-      quality
-    })
-
-    stream.label = label
-    stream.updateTitle().updateFilepath()
-
-    streams.add(stream)
-    processedIssues.add(issue)
   }
 }
+
+async function removeStream(issue: Issue) {
+  const log = createThread(issue, 'streams/remove')
+  log.start()
+
+  const data = issue.data
+  if (data.missing('stream_url')) return
+
+  const streamUrls = data.getString('stream_url') || ''
+
+  let changed = false
+  streamUrls
+    .split(/\r?\n/)
+    .filter(Boolean)
+    .forEach(link => {
+      const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim())
+      if (found) {
+        found.removed = true
+        changed = true
+      }
+    })
+
+  if (changed) processedIssues.add(issue)
+}
+
+async function editStream(issue: Issue) {
+  const data = issue.data
+
+  if (data.missing('stream_url')) return
+
+  const stream: Stream = streams.first(
+    (_stream: Stream) => _stream.url === data.getString('stream_url')
+  )
+  if (!stream) return
+
+  const streamId = data.getString('stream_id') || ''
+  const [channelId, feedId] = streamId.split('@')
+
+  if (channelId) {
+    stream.channel = channelId
+    stream.feed = feedId
+    stream.updateTvgId().updateTitle().updateFilepath()
+  }
+
+  stream.updateWithIssue(data)
+
+  processedIssues.add(issue)
+}
+
+async function addStream(issue: Issue) {
+  const data = issue.data
+  if (data.missing('stream_id') || data.missing('stream_url')) return
+  if (streams.includes((_stream: Stream) => _stream.url === data.getString('stream_url'))) return
+  const streamUrl = data.getString('stream_url') || ''
+  if (!isURI(streamUrl)) return
+
+  const streamId = data.getString('stream_id') || ''
+  const [channelId, feedId] = streamId.split('@')
+
+  const channel: sdk.Models.Channel | undefined = apiData.channelsKeyById.get(channelId)
+  if (!channel) return
+
+  const label = data.getString('label') || ''
+  const httpUserAgent = data.getString('http_user_agent') || null
+  const httpReferrer = data.getString('http_referrer') || null
+
+  let quality = data.getString('quality') || null
+  if (!quality) {
+    const streamInfo = await getStreamInfo(streamUrl, { httpUserAgent, httpReferrer })
+
+    if (streamInfo) {
+      const height = streamInfo?.resolution?.height
+
+      if (height) {
+        quality = `${height}p`
+      }
+    }
+  }
+
+  const stream = new Stream({
+    channel: channelId,
+    feed: feedId,
+    title: channel.name,
+    url: streamUrl,
+    user_agent: httpUserAgent,
+    referrer: httpReferrer,
+    quality
+  })
+
+  stream.label = label
+  stream.updateTitle().updateFilepath()
+
+  streams.add(stream)
+  processedIssues.add(issue)
+}

scripts/utils.ts

@@ -167,6 +167,7 @@ export async function loadIssues(props?: { labels: string | string[] }) {
       per_page: 100,
       labels,
       status: 'open',
+      direction: 'asc',
       headers: {
         'X-GitHub-Api-Version': '2022-11-28'
       }
@@ -293,3 +294,33 @@ function parseDiscussion(discussion: {
     data: new DataSet(data)
   })
 }
+
+class LogThread {
+  issue: Issue
+  type: string
+
+  constructor(issue: Issue, type: string) {
+    this.issue = issue
+    this.type = type
+  }
+
+  start() {
+    console.log(`[#${this.issue.number}] ${this.type}: Issue #${this.issue.number}`)
+  }
+
+  warn(message: string) {
+    console.log(`[#${this.issue.number}] ${this.type}: └── WARNING: ${message}`)
+  }
+
+  error(message: string) {
+    console.log(`[#${this.issue.number}] ${this.type}: └── ERROR: ${message}`)
+  }
+
+  info(message: string) {
+    console.log(`[#${this.issue.number}] ${this.type}: └── INFO: ${message}`)
+  }
+}
+
+export function createThread(issue: Issue, type: string): LogThread {
+  return new LogThread(issue, type)
+}

tests/__data__/expected/playlist_update/playlist_update.log

@@ -1 +1 @@
-closes #14151, closes #14150, closes #14110, closes #14120, closes #14175, closes #14105, closes #14104, closes #14057, closes #14034, closes #13964, closes #13893, closes #13881, closes #13793, closes #13751, closes #13715
+closes #14175, closes #14105, closes #14104, closes #14057, closes #14034, closes #13964, closes #13893, closes #13881, closes #13793, closes #13751, closes #13715, closes #14110, closes #14120, closes #14151, closes #14150