Merge pull request #12654 from iptv-org/patch-2023.05.2

Update scripts

scripts/commands/database/create.js

@@ -18,12 +18,12 @@ async function main() {
 
       const stream = store.create()
 
-      stream.set('channel', { channel: item.tvg.id })
-      stream.set('title', { title: item.name })
-      stream.set('filepath', { filepath: item.filepath })
-      stream.set('url', { url: item.url })
-      stream.set('http_referrer', { http_referrer: item.http.referrer })
-      stream.set('user_agent', { user_agent: item.http['user-agent'] })
+      stream.set('channel', item.tvg.id)
+      stream.set('title', item.name)
+      stream.set('filepath', item.filepath)
+      stream.set('url', item.url)
+      stream.set('http_referrer', item.http.referrer)
+      stream.set('user_agent', item.http['user-agent'])
 
       streams.push(stream)
     }

scripts/commands/playlist/format.js

@@ -1,4 +1,5 @@
 const { create: createPlaylist } = require('../../core/playlist')
+const { normalize: normalizeUrl } = require('../../core/url')
 const { db, logger, file } = require('../../core')
 const { orderBy } = require('natural-orderby')
 const _ = require('lodash')
@@ -8,6 +9,12 @@ async function main() {
   await db.streams.load()
   let streams = await db.streams.find({})
 
+  streams = streams.map(stream => {
+    stream.url = normalizeUrl(stream.url)
+
+    return stream
+  })
+
   logger.info('sorting links...')
   streams = orderBy(
     streams,

scripts/commands/playlist/validate.js

@@ -30,6 +30,7 @@ async function main() {
     const basename = file.basename(filepath)
     const [__, country] = basename.match(/([a-z]{2})(|_.*)\.m3u/i) || [null, null]
 
+    const buffer = {}
     const fileLog = []
     const playlist = await parser.parsePlaylist(filepath)
     for (const item of playlist.items) {
@@ -41,6 +42,16 @@ async function main() {
         })
       }
 
+      if (item.url && buffer[item.url]) {
+        fileLog.push({
+          type: 'warning',
+          line: item.line,
+          message: `"${item.url}" is already on the playlist`
+        })
+      } else {
+        buffer[item.url] = true
+      }
+
       const channel_id = id.generate(item.name, country)
       const found = blocklist.find(
         blocked =>

scripts/core/url.js

@@ -0,0 +1,11 @@
+const normalize = require('normalize-url')
+
+const url = {}
+
+url.normalize = function (string) {
+  const normalized = normalize(string, { stripWWW: false })
+
+  return decodeURIComponent(normalized).replace(/\s/g, '+')
+}
+
+module.exports = url

scripts/store/setters/channel.js

@@ -1,3 +1,3 @@
-module.exports = function ({ channel }) {
+module.exports = function (channel) {
   return channel || null
 }

scripts/store/setters/http_referrer.js

@@ -1,3 +1,3 @@
-module.exports = function ({ http_referrer }) {
+module.exports = function (http_referrer) {
   return http_referrer || null
 }

scripts/store/setters/index.js

@@ -1,4 +1,3 @@
-exports.url = require('./url')
 exports.http_referrer = require('./http_referrer')
 exports.user_agent = require('./user_agent')
 exports.channel = require('./channel')

scripts/store/setters/url.js

@@ -1,7 +0,0 @@
-const normalize = require('normalize-url')
-
-module.exports = function ({ url }) {
-  const normalized = normalize(url, { stripWWW: false })
-
-  return decodeURIComponent(normalized).replace(/\s/g, '+')
-}

scripts/store/setters/user_agent.js

@@ -1,3 +1,3 @@
-module.exports = function ({ user_agent }) {
+module.exports = function (user_agent) {
   return user_agent || null
 }

tests/__data__/expected/streams/ru.m3u

@@ -6,4 +6,4 @@ http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8
 #EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p)
 http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8
 #EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p)
-http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8
+https://service-stitcher.clusters.pluto.tv/stitch/hls/channel/5ca525b650be2571e3943c63/master.m3u8?advertisingId=&appName=web&deviceId=5ca525b650be2571e3943c63

tests/__data__/input/database/playlist_format.streams.db

@@ -1,4 +1,4 @@
-{"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF3"}
+{"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"https://service-stitcher.clusters.pluto.tv/stitch/hls/channel/5ca525b650be2571e3943c63/master.m3u8?deviceId=5ca525b650be2571e3943c63&appName=web&advertisingId=","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF3"}
 {"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF0"}
 {"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF1"}
 {"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF2"}