Revert "Update "format" script"

scripts/clean.js

@@ -0,0 +1,74 @@
+const IPTVChecker = require('iptv-checker')
+const { program } = require('commander')
+const ProgressBar = require('progress')
+const parser = require('./helpers/parser')
+const utils = require('./helpers/utils')
+const log = require('./helpers/log')
+
+program
+  .usage('[OPTIONS]...')
+  .option('-d, --debug', 'Enable debug mode')
+  .option('-c, --country <country>', 'Comma-separated list of country codes', '')
+  .option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
+  .option('--timeout <timeout>', 'Set timeout for each request', 5000)
+  .parse(process.argv)
+
+let bar
+const config = program.opts()
+const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline']
+const checker = new IPTVChecker({
+  timeout: config.timeout
+})
+
+async function main() {
+  log.start()
+
+  if (config.debug) log.print(`Debug mode enabled\n`)
+
+  let playlists = parser.parseIndex()
+  playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
+  for (const playlist of playlists) {
+    await parser
+      .parsePlaylist(playlist.url)
+      .then(checkPlaylist)
+      .then(p => p.save())
+  }
+
+  log.finish()
+}
+
+async function checkPlaylist(playlist) {
+  if (!config.debug) {
+    bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, {
+      total: playlist.channels.length
+    })
+  }
+  const channels = []
+  const total = playlist.channels.length
+  for (const [index, channel] of playlist.channels.entries()) {
+    const skipChannel =
+      channel.status &&
+      ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
+    if (skipChannel) {
+      channels.push(channel)
+    } else {
+      const result = await checker.checkStream(channel.data)
+      if (result.status.ok || result.status.reason.includes('timed out')) {
+        channels.push(channel)
+      } else {
+        if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`)
+      }
+    }
+    if (!config.debug) bar.tick()
+  }
+
+  if (playlist.channels.length !== channels.length) {
+    log.print(`File '${playlist.url}' has been updated\n`)
+    playlist.channels = channels
+    playlist.updated = true
+  }
+
+  return playlist
+}
+
+main()

scripts/detect-resolution.js

@@ -0,0 +1,114 @@
+const { program } = require('commander')
+const ProgressBar = require('progress')
+const axios = require('axios')
+const https = require('https')
+const parser = require('./helpers/parser')
+const utils = require('./helpers/utils')
+const log = require('./helpers/log')
+
+program
+  .usage('[OPTIONS]...')
+  .option('-c, --country <country>', 'Comma-separated list of country codes', '')
+  .option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
+  .option('--delay <delay>', 'Delay between parser requests', 1000)
+  .option('--timeout <timeout>', 'Set timeout for each request', 5000)
+  .parse(process.argv)
+
+const config = program.opts()
+const ignoreStatus = ['Offline']
+const instance = axios.create({
+  timeout: config.timeout,
+  maxContentLength: 200000,
+  httpsAgent: new https.Agent({
+    rejectUnauthorized: false
+  })
+})
+
+async function main() {
+  log.start()
+
+  log.print(`Parsing 'index.m3u'...\n`)
+  let playlists = parser.parseIndex()
+  playlists = utils
+    .filterPlaylists(playlists, config.country, config.exclude)
+    .filter(i => i.url !== 'channels/unsorted.m3u')
+
+  for (const playlist of playlists) {
+    await parser
+      .parsePlaylist(playlist.url)
+      .then(detectResolution)
+      .then(p => p.save())
+  }
+
+  log.finish()
+}
+
+async function detectResolution(playlist) {
+  const channels = []
+  const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, {
+    total: playlist.channels.length
+  })
+  let updated = false
+  for (const channel of playlist.channels) {
+    bar.tick()
+    const skipChannel =
+      channel.status &&
+      ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
+    if (!channel.resolution.height && !skipChannel) {
+      const CancelToken = axios.CancelToken
+      const source = CancelToken.source()
+      const timeout = setTimeout(() => {
+        source.cancel()
+      }, config.timeout)
+
+      const response = await instance
+        .get(channel.url, { cancelToken: source.token })
+        .then(res => {
+          clearTimeout(timeout)
+
+          return res
+        })
+        .then(utils.sleep(config.delay))
+        .catch(err => {
+          clearTimeout(timeout)
+        })
+
+      if (response && response.status === 200) {
+        if (/^#EXTM3U/.test(response.data)) {
+          const resolution = parseResolution(response.data)
+          if (resolution) {
+            channel.resolution = resolution
+            updated = true
+          }
+        }
+      }
+    }
+
+    channels.push(channel)
+  }
+
+  if (updated) {
+    log.print(`File '${playlist.url}' has been updated\n`)
+    playlist.channels = channels
+    playlist.updated = true
+  }
+
+  return playlist
+}
+
+function parseResolution(string) {
+  const regex = /RESOLUTION=(\d+)x(\d+)/gm
+  const match = string.matchAll(regex)
+  const arr = Array.from(match).map(m => ({
+    width: parseInt(m[1]),
+    height: parseInt(m[2])
+  }))
+
+  return arr.length
+    ? arr.reduce(function (prev, current) {
+        return prev.height > current.height ? prev : current
+      })
+    : undefined
+}
+
+main()

scripts/format.js

@@ -1,46 +1,21 @@
-const IPTVChecker = require('iptv-checker')
-const normalize = require('normalize-url')
-const { program } = require('commander')
-const ProgressBar = require('progress')
 const parser = require('./helpers/parser')
 const utils = require('./helpers/utils')
 const file = require('./helpers/file')
 const log = require('./helpers/log')
 
-program
-  .usage('[OPTIONS]...')
-  .option('-d, --debug', 'Enable debug mode')
-  .option('-s, --status', 'Update stream status')
-  .option('-r, --resolution', 'Detect stream resolution')
-  .option('-c, --country <country>', 'Comma-separated list of country codes', '')
-  .option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
-  .option('--timeout <timeout>', 'Set timeout for each request', 5000)
-  .parse(process.argv)
-
-let bar
-const ignoreStatus = ['Geo-blocked', 'Not 24/7']
-const config = program.opts()
-const checker = new IPTVChecker({
-  timeout: config.timeout
-})
-
 async function main() {
   log.start()
 
-  if (config.debug) log.print(`Debug mode enabled\n`)
-  if (config.status) log.print(`Status check enabled\n`)
-  if (config.resolution) log.print(`Resolution detection enabled\n`)
-
+  log.print(`Parsing 'index.m3u'...`)
   let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
-  playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
-  if (!playlists.length) log.print(`No playlist is selected\n`)
   for (const playlist of playlists) {
+    log.print(`\nProcessing '${playlist.url}'...`)
     await parser
       .parsePlaylist(playlist.url)
-      .then(updatePlaylist)
+      .then(formatPlaylist)
       .then(playlist => {
         if (file.read(playlist.url) !== playlist.toString()) {
-          log.print(`File '${playlist.url}' has been updated\n`)
+          log.print('updated')
           playlist.updated = true
         }
 
@@ -48,115 +23,33 @@ async function main() {
       })
   }
 
+  log.print('\n')
   log.finish()
 }
 
-async function updatePlaylist(playlist) {
-  if (!config.debug) {
-    bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, {
-      total: playlist.channels.length
-    })
-  } else {
-    log.print(`Processing '${playlist.url}'...\n`)
-  }
-
+async function formatPlaylist(playlist) {
   for (const channel of playlist.channels) {
-    addMissingData(channel)
-    updateGroupTitle(channel)
-    normalizeUrl(channel)
-
-    const checkOnline = config.status || config.resolution
-    const skipChannel =
-      channel.status &&
-      ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
-    if (checkOnline && !skipChannel) {
-      await checker
-        .checkStream(channel.data)
-        .then(result => {
-          const status = parseStatus(result.status)
-
-          if (config.status) {
-            updateStatus(channel, status)
-          }
-
-          if (config.resolution && status === 'online') {
-            updateResolution(channel, result.status.metadata)
-          }
-
-          if (config.debug && status === 'offline') {
-            log.print(`  ERR: ${channel.url} (${result.status.reason})\n`)
-          }
-        })
-        .catch(err => {
-          if (config.debug) log.print(`  ERR: ${channel.url} (${err.message})\n`)
-        })
+    const code = file.getBasename(playlist.url)
+    // add missing tvg-name
+    if (!channel.tvg.name && code !== 'unsorted' && channel.name) {
+      channel.tvg.name = channel.name.replace(/\"/gi, '')
     }
-    if (!config.debug) bar.tick()
+    // add missing tvg-id
+    if (!channel.tvg.id && code !== 'unsorted' && channel.tvg.name) {
+      const id = utils.name2id(channel.tvg.name)
+      channel.tvg.id = id ? `${id}.${code}` : ''
+    }
+    // add missing country
+    if (!channel.countries.length) {
+      const name = utils.code2name(code)
+      channel.countries = name ? [{ code, name }] : []
+      channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';')
+    }
+    // update group-title
+    channel.group.title = channel.category
   }
 
   return playlist
 }
 
-function parseStatus(status) {
-  if (status.ok) {
-    return 'online'
-  } else if (status.reason.includes('timed out')) {
-    return 'timeout'
-  } else if (status.reason.includes('403')) {
-    return 'error_403'
-  } else if (status.reason.includes('not one of 40{0,1,3,4}')) {
-    return 'error_40x' // 402, 451
-  } else {
-    return 'offline'
-  }
-}
-
-function updateStatus(channel, status) {
-  switch (status) {
-    case 'online':
-      channel.status = null
-      break
-    case 'offline':
-      channel.status = 'Offline'
-      break
-  }
-}
-
-function addMissingData(channel) {
-  // tvg-name
-  if (!channel.tvg.name && channel.name) {
-    channel.tvg.name = channel.name.replace(/\"/gi, '')
-  }
-  // tvg-id
-  if (!channel.tvg.id && channel.tvg.name) {
-    const id = utils.name2id(channel.tvg.name)
-    channel.tvg.id = id ? `${id}.${code}` : ''
-  }
-  // country
-  if (!channel.countries.length) {
-    const name = utils.code2name(code)
-    channel.countries = name ? [{ code, name }] : []
-    channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';')
-  }
-}
-
-function updateGroupTitle(channel) {
-  channel.group.title = channel.category
-}
-
-function normalizeUrl(channel) {
-  const normalized = normalize(channel.url, { stripWWW: false })
-  channel.updateUrl(normalized)
-}
-
-function updateResolution(channel, metadata) {
-  const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : []
-  if (!channel.resolution.height && streams.length) {
-    channel.resolution = streams.reduce((acc, curr) => {
-      if (curr.height > acc.height) return { width: curr.width, height: curr.height }
-      return acc
-    })
-  }
-}
-
 main()

scripts/helpers/Channel.js

@@ -22,11 +22,6 @@ module.exports = class Channel {
     this.languages = this.parseLanguages(data.tvg.language)
   }
 
-  updateUrl(url) {
-    this.url = url
-    this.data.url = url
-  }
-
   parseName(title) {
     return title
       .trim()

scripts/helpers/utils.js

@@ -1,3 +1,4 @@
+const escapeStringRegexp = require('escape-string-regexp')
 const transliteration = require('transliteration')
 const iso6393 = require('@freearhey/iso-639-3')
 const categories = require('./categories')
@@ -70,6 +71,16 @@ utils.sortBy = function (arr, fields) {
   })
 }
 
+utils.escapeStringRegexp = function (scring) {
+  return escapeStringRegexp(string)
+}
+
+utils.sleep = function (ms) {
+  return function (x) {
+    return new Promise(resolve => setTimeout(() => resolve(x), ms))
+  }
+}
+
 utils.removeProtocol = function (string) {
   return string.replace(/(^\w+:|^)\/\//, '')
 }