merge patch 1/7 to the rest, patch validate to take multiple files

scripts/core/channelsParser.ts

@@ -1,5 +1,6 @@
 import { parseChannels } from 'epg-grabber'
-import { Storage, Collection } from '@freearhey/core'
+import { Storage } from '@freearhey/core'
+import { ChannelList } from '../models'
 
 type ChannelsParserProps = {
   storage: Storage
@@ -12,13 +13,10 @@ export class ChannelsParser {
     this.storage = storage
   }
 
-  async parse(filepath: string) {
-    let parsedChannels = new Collection()
-
+  async parse(filepath: string): Promise<ChannelList> {
     const content = await this.storage.load(filepath)
-    const channels = parseChannels(content)
-    parsedChannels = parsedChannels.concat(new Collection(channels))
+    const parsed = parseChannels(content)
 
-    return parsedChannels
+    return new ChannelList({ channels: parsed })
   }
 }

scripts/core/dataLoader.ts

@@ -49,7 +49,8 @@ export class DataLoader {
       feeds,
       timezones,
       guides,
-      streams
+      streams,
+      logos
     ] = await Promise.all([
       this.storage.json('countries.json'),
       this.storage.json('regions.json'),
@@ -61,7 +62,8 @@ export class DataLoader {
       this.storage.json('feeds.json'),
       this.storage.json('timezones.json'),
       this.storage.json('guides.json'),
-      this.storage.json('streams.json')
+      this.storage.json('streams.json'),
+      this.storage.json('logos.json')
     ])
 
     return {
@@ -75,7 +77,8 @@ export class DataLoader {
       feeds,
       timezones,
       guides,
-      streams
+      streams,
+      logos
     }
   }
 

scripts/core/dataProcessor.ts

@@ -1,6 +1,6 @@
+import { Channel, Feed, GuideChannel, Logo, Stream } from '../models'
 import { DataLoaderData } from '../types/dataLoader'
 import { Collection } from '@freearhey/core'
-import { Channel, Feed, Guide, Stream } from '../models'
 
 export class DataProcessor {
   constructor() {}
@@ -9,31 +9,48 @@ export class DataProcessor {
     let channels = new Collection(data.channels).map(data => new Channel(data))
     const channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
 
-    const guides = new Collection(data.guides).map(data => new Guide(data))
-    const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId())
+    const guideChannels = new Collection(data.guides).map(data => new GuideChannel(data))
+    const guideChannelsGroupedByStreamId = guideChannels.groupBy((channel: GuideChannel) =>
+      channel.getStreamId()
+    )
 
     const streams = new Collection(data.streams).map(data => new Stream(data))
     const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
 
-    const feeds = new Collection(data.feeds).map(data =>
+    let feeds = new Collection(data.feeds).map(data =>
       new Feed(data)
-        .withGuides(guidesGroupedByStreamId)
+        .withGuideChannels(guideChannelsGroupedByStreamId)
         .withStreams(streamsGroupedById)
         .withChannel(channelsKeyById)
     )
+    const feedsKeyByStreamId = feeds.keyBy((feed: Feed) => feed.getStreamId())
+
+    const logos = new Collection(data.logos).map(data =>
+      new Logo(data).withFeed(feedsKeyByStreamId)
+    )
+    const logosGroupedByChannelId = logos.groupBy((logo: Logo) => logo.channelId)
+    const logosGroupedByStreamId = logos.groupBy((logo: Logo) => logo.getStreamId())
+
+    feeds = feeds.map((feed: Feed) => feed.withLogos(logosGroupedByStreamId))
     const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
 
-    channels = channels.map((channel: Channel) => channel.withFeeds(feedsGroupedByChannelId))
+    channels = channels.map((channel: Channel) =>
+      channel.withFeeds(feedsGroupedByChannelId).withLogos(logosGroupedByChannelId)
+    )
 
     return {
+      guideChannelsGroupedByStreamId,
       feedsGroupedByChannelId,
-      guidesGroupedByStreamId,
+      logosGroupedByChannelId,
+      logosGroupedByStreamId,
       streamsGroupedById,
+      feedsKeyByStreamId,
       channelsKeyById,
+      guideChannels,
       channels,
       streams,
-      guides,
-      feeds
+      feeds,
+      logos
     }
   }
 }

scripts/core/grabber.ts

@@ -70,6 +70,10 @@ export class Grabber {
               }
             }
 
+            if (this.options.curl === true) {
+              config.curl = true
+            }
+
             const _programs = await this.grabber.grab(
               channel,
               date,

scripts/core/guide.ts

@@ -1,60 +0,0 @@
-import { Collection, Logger, DateTime, Storage, Zip } from '@freearhey/core'
-import { Channel } from 'epg-grabber'
-import { XMLTV } from '../core'
-import path from 'path'
-
-type GuideProps = {
-  channels: Collection
-  programs: Collection
-  logger: Logger
-  filepath: string
-  gzip: boolean
-}
-
-export class Guide {
-  channels: Collection
-  programs: Collection
-  logger: Logger
-  storage: Storage
-  filepath: string
-  gzip: boolean
-
-  constructor({ channels, programs, logger, filepath, gzip }: GuideProps) {
-    this.channels = channels
-    this.programs = programs
-    this.logger = logger
-    this.storage = new Storage(path.dirname(filepath))
-    this.filepath = filepath
-    this.gzip = gzip || false
-  }
-
-  async save() {
-    const channels = this.channels.uniqBy(
-      (channel: Channel) => `${channel.xmltv_id}:${channel.site}`
-    )
-    const programs = this.programs
-
-    const currDate = new DateTime(process.env.CURR_DATE || new Date().toISOString(), {
-      timezone: 'UTC'
-    })
-    const xmltv = new XMLTV({
-      channels,
-      programs,
-      date: currDate
-    })
-
-    const xmlFilepath = this.filepath
-    const xmlFilename = path.basename(xmlFilepath)
-    this.logger.info(`  saving to "${xmlFilepath}"...`)
-    await this.storage.save(xmlFilename, xmltv.toString())
-
-    if (this.gzip) {
-      const zip = new Zip()
-      const compressed = zip.compress(xmltv.toString())
-      const gzFilepath = `${this.filepath}.gz`
-      const gzFilename = path.basename(gzFilepath)
-      this.logger.info(`  saving to "${gzFilepath}"...`)
-      await this.storage.save(gzFilename, compressed)
-    }
-  }
-}

scripts/core/guideManager.ts

@@ -1,7 +1,12 @@
-import { Collection, Logger, Storage, StringTemplate } from '@freearhey/core'
+import { Collection, Logger, Zip, Storage, StringTemplate } from '@freearhey/core'
+import epgGrabber from 'epg-grabber'
 import { OptionValues } from 'commander'
-import { Channel, Program } from 'epg-grabber'
-import { Guide } from '.'
+import { Channel, Feed, Guide } from '../models'
+import path from 'path'
+import { DataLoader, DataProcessor } from '.'
+import { DataLoaderData } from '../types/dataLoader'
+import { DataProcessorData } from '../types/dataProcessor'
+import { DATA_DIR } from '../constants'
 
 type GuideManagerProps = {
   options: OptionValues
@@ -12,7 +17,6 @@ type GuideManagerProps = {
 
 export class GuideManager {
   options: OptionValues
-  storage: Storage
   logger: Logger
   channels: Collection
   programs: Collection
@@ -22,22 +26,51 @@ export class GuideManager {
     this.logger = logger
     this.channels = channels
     this.programs = programs
-    this.storage = new Storage()
   }
 
   async createGuides() {
     const pathTemplate = new StringTemplate(this.options.output)
 
+    const processor = new DataProcessor()
+    const dataStorage = new Storage(DATA_DIR)
+    const loader = new DataLoader({ storage: dataStorage })
+    const data: DataLoaderData = await loader.load()
+    const { feedsKeyByStreamId, channelsKeyById }: DataProcessorData = processor.process(data)
+
     const groupedChannels = this.channels
-      .orderBy([(channel: Channel) => channel.xmltv_id])
-      .uniqBy((channel: Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`)
-      .groupBy((channel: Channel) => {
+      .map((channel: epgGrabber.Channel) => {
+        if (channel.xmltv_id && !channel.icon) {
+          const foundFeed: Feed = feedsKeyByStreamId.get(channel.xmltv_id)
+          if (foundFeed && foundFeed.hasLogo()) {
+            channel.icon = foundFeed.getLogoUrl()
+          } else {
+            const [channelId] = channel.xmltv_id.split('@')
+            const foundChannel: Channel = channelsKeyById.get(channelId)
+            if (foundChannel && foundChannel.hasLogo()) {
+              channel.icon = foundChannel.getLogoUrl()
+            }
+          }
+        }
+
+        return channel
+      })
+      .orderBy([
+        (channel: epgGrabber.Channel) => channel.index,
+        (channel: epgGrabber.Channel) => channel.xmltv_id
+      ])
+      .uniqBy(
+        (channel: epgGrabber.Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`
+      )
+      .groupBy((channel: epgGrabber.Channel) => {
         return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' })
       })
 
     const groupedPrograms = this.programs
-      .orderBy([(program: Program) => program.channel, (program: Program) => program.start])
-      .groupBy((program: Program) => {
+      .orderBy([
+        (program: epgGrabber.Program) => program.channel,
+        (program: epgGrabber.Program) => program.start
+      ])
+      .groupBy((program: epgGrabber.Program) => {
         const lang =
           program.titles && program.titles.length && program.titles[0].lang
             ? program.titles[0].lang
@@ -51,11 +84,28 @@ export class GuideManager {
         filepath: groupKey,
         gzip: this.options.gzip,
         channels: new Collection(groupedChannels.get(groupKey)),
-        programs: new Collection(groupedPrograms.get(groupKey)),
-        logger: this.logger
+        programs: new Collection(groupedPrograms.get(groupKey))
       })
 
-      await guide.save()
+      await this.save(guide)
+    }
+  }
+
+  async save(guide: Guide) {
+    const storage = new Storage(path.dirname(guide.filepath))
+    const xmlFilepath = guide.filepath
+    const xmlFilename = path.basename(xmlFilepath)
+    this.logger.info(`  saving to "${xmlFilepath}"...`)
+    const xmltv = guide.toString()
+    await storage.save(xmlFilename, xmltv)
+
+    if (guide.gzip) {
+      const zip = new Zip()
+      const compressed = zip.compress(xmltv)
+      const gzFilepath = `${guide.filepath}.gz`
+      const gzFilename = path.basename(gzFilepath)
+      this.logger.info(`  saving to "${gzFilepath}"...`)
+      await storage.save(gzFilename, compressed)
     }
   }
 }

scripts/core/index.ts

@@ -4,7 +4,6 @@ export * from './configLoader'
 export * from './dataLoader'
 export * from './dataProcessor'
 export * from './grabber'
-export * from './guide'
 export * from './guideManager'
 export * from './htmlTable'
 export * from './issueLoader'
@@ -13,5 +12,3 @@ export * from './job'
 export * from './proxyParser'
 export * from './queue'
 export * from './queueCreator'
-export * from './xml'
-export * from './xmltv'

scripts/core/issueLoader.ts

@@ -23,6 +23,7 @@ export class IssueLoader {
         repo: REPO,
         per_page: 100,
         labels,
+        state: 'open',
         headers: {
           'X-GitHub-Api-Version': '2022-11-28'
         }

scripts/core/proxyParser.ts

@@ -2,9 +2,9 @@ import { URL } from 'node:url'
 
 type ProxyParserResult = {
   protocol: string | null
-  auth: {
-    username: string | null
-    password: string | null
+  auth?: {
+    username?: string
+    password?: string
   }
   host: string
   port: number | null
@@ -14,14 +14,18 @@ export class ProxyParser {
   parse(_url: string): ProxyParserResult {
     const parsed = new URL(_url)
 
-    return {
+    const result: ProxyParserResult = {
       protocol: parsed.protocol.replace(':', '') || null,
-      auth: {
-        username: parsed.username || null,
-        password: parsed.password || null
-      },
       host: parsed.hostname,
       port: parsed.port ? parseInt(parsed.port) : null
     }
+
+    if (parsed.username || parsed.password) {
+      result.auth = {}
+      if (parsed.username) result.auth.username = parsed.username
+      if (parsed.password) result.auth.password = parsed.password
+    }
+
+    return result
   }
 }

scripts/core/queueCreator.ts

@@ -1,15 +1,14 @@
 import { Storage, Collection, DateTime, Logger } from '@freearhey/core'
-import { ChannelsParser, ConfigLoader, Queue } from './'
 import { SITES_DIR, DATA_DIR } from '../constants'
+import { GrabOptions } from '../commands/epg/grab'
+import { ConfigLoader, Queue } from './'
 import { SiteConfig } from 'epg-grabber'
 import path from 'path'
-import { GrabOptions } from '../commands/epg/grab'
-import { Channel } from '../models'
 
 type QueueCreatorProps = {
   logger: Logger
   options: GrabOptions
-  parsedChannels: Collection
+  channels: Collection
 }
 
 export class QueueCreator {
@@ -17,42 +16,29 @@ export class QueueCreator {
   logger: Logger
   sitesStorage: Storage
   dataStorage: Storage
-  parser: ChannelsParser
-  parsedChannels: Collection
+  channels: Collection
   options: GrabOptions
 
-  constructor({ parsedChannels, logger, options }: QueueCreatorProps) {
-    this.parsedChannels = parsedChannels
+  constructor({ channels, logger, options }: QueueCreatorProps) {
+    this.channels = channels
     this.logger = logger
     this.sitesStorage = new Storage()
     this.dataStorage = new Storage(DATA_DIR)
-    this.parser = new ChannelsParser({ storage: new Storage() })
     this.options = options
     this.configLoader = new ConfigLoader()
   }
 
   async create(): Promise<Queue> {
-    const channelsContent = await this.dataStorage.json('channels.json')
-    const channels = new Collection(channelsContent).map(data => new Channel(data))
-
+    let index = 0
     const queue = new Queue()
-    for (const channel of this.parsedChannels.all()) {
+    for (const channel of this.channels.all()) {
+      channel.index = index++
       if (!channel.site || !channel.site_id || !channel.name) continue
 
       const configPath = path.resolve(SITES_DIR, `${channel.site}/${channel.site}.config.js`)
       const config: SiteConfig = await this.configLoader.load(configPath)
 
-      if (channel.xmltv_id) {
-        if (!channel.icon) {
-          const found: Channel = channels.first(
-            (_channel: Channel) => _channel.id === channel.xmltv_id
-          )
-
-          if (found) {
-            channel.icon = found.logo
-          }
-        }
-      } else {
+      if (!channel.xmltv_id) {
         channel.xmltv_id = channel.site_id
       }
 

scripts/core/xml.ts

@@ -1,56 +0,0 @@
-import { Collection } from '@freearhey/core'
-import { Channel } from 'epg-grabber'
-
-export class XML {
-  items: Collection
-
-  constructor(items: Collection) {
-    this.items = items
-  }
-
-  toString() {
-    let output = '<?xml version="1.0" encoding="UTF-8"?>\r\n<channels>\r\n'
-
-    this.items.forEach((channel: Channel) => {
-      const logo = channel.logo ? ` logo="${channel.logo}"` : ''
-      const xmltv_id = channel.xmltv_id || ''
-      const lang = channel.lang || ''
-      const site_id = channel.site_id || ''
-      output += `  <channel site="${channel.site}" lang="${lang}" xmltv_id="${escapeString(
-        xmltv_id
-      )}" site_id="${site_id}"${logo}>${escapeString(channel.name)}</channel>\r\n`
-    })
-
-    output += '</channels>\r\n'
-
-    return output
-  }
-}
-
-function escapeString(value: string, defaultValue: string = '') {
-  if (!value) return defaultValue
-
-  const regex = new RegExp(
-    '((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
-      'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
-      'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
-      '|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
-      'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
-      '[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
-      'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
-      '(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
-    'g'
-  )
-
-  value = String(value || '').replace(regex, '')
-
-  return value
-    .replace(/&/g, '&amp;')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-    .replace(/'/g, '&apos;')
-    .replace(/\n|\r/g, ' ')
-    .replace(/  +/g, ' ')
-    .trim()
-}

scripts/core/xmltv.ts

@@ -1,28 +0,0 @@
-import { DateTime, Collection } from '@freearhey/core'
-import { generateXMLTV } from 'epg-grabber'
-
-type XMLTVProps = {
-  channels: Collection
-  programs: Collection
-  date: DateTime
-}
-
-export class XMLTV {
-  channels: Collection
-  programs: Collection
-  date: DateTime
-
-  constructor({ channels, programs, date }: XMLTVProps) {
-    this.channels = channels
-    this.programs = programs
-    this.date = date
-  }
-
-  toString() {
-    return generateXMLTV({
-      channels: this.channels.all(),
-      programs: this.programs.all(),
-      date: this.date.toJSON()
-    })
-  }
-}