diff --git a/scripts/commands/issue/validate.ts b/scripts/commands/issue/validate.ts index bd7acecd51..924f164c55 100644 --- a/scripts/commands/issue/validate.ts +++ b/scripts/commands/issue/validate.ts @@ -84,6 +84,20 @@ async function main() { errors = errors.concat(validateStreamId(streamId)) } } + + if ( + data.missing([ + 'stream_id', + 'quality', + 'live_247', + 'geo_blocked', + 'http_user_agent', + 'http_referrer' + ]) + ) { + errors.push('The request does not contain any changes') + done() + } } done() diff --git a/scripts/commands/playlist/format.ts b/scripts/commands/playlist/format.ts index 47f6c85fbf..45986c0a01 100644 --- a/scripts/commands/playlist/format.ts +++ b/scripts/commands/playlist/format.ts @@ -128,7 +128,7 @@ async function main() { if (streams.isEmpty()) return - const playlist = new Playlist(streams, { public: false }) + const playlist = new Playlist(streams, { public: false, raw: true }) await streamsStorage.save(filepath, playlist.toString()) } } diff --git a/scripts/commands/playlist/generate.ts b/scripts/commands/playlist/generate.ts index 6fcb0cf18d..d65c1164fb 100644 --- a/scripts/commands/playlist/generate.ts +++ b/scripts/commands/playlist/generate.ts @@ -44,20 +44,35 @@ async function main() { logger.info('generating raw/...') await new RawGenerator({ streams, logFile }).generate() + logger.info('create unique names...') + const channelCountries = new Map>() + streams.forEach((stream: Stream) => { + if (!channelCountries.has(stream.channelName)) { + channelCountries.set(stream.channelName, new Set()) + } + channelCountries.get(stream.channelName)!.add(stream.countryName) + }) + for (const stream of streams.all()) { + const countries = channelCountries.get(stream.channelName) + stream.hasUniqueName = countries ? countries.size === 1 : false + } + logger.info('sorting streams...') streams = streams.sortBy( [ - (stream: Stream) => stream.getId(), + (stream: Stream) => stream.channelUniqueName, + (stream: Stream) => (stream.hasMainFeed ? 1 : 0), + (stream: Stream) => stream.feedName, (stream: Stream) => (stream.isGeoBlocked ? -1 : 0), (stream: Stream) => (stream.isNot247 ? -1 : 0), (stream: Stream) => stream.getVerticalResolution() ], - ['asc', 'desc', 'desc', 'desc'] + ['asc', 'desc', 'asc', 'desc', 'desc', 'desc'] ) logger.info('filtering streams...') streams = streams - .filter((stream: Stream) => !!stream.getId()) + .filter((stream: Stream) => stream.hasChannel() && stream.hasFeed()) .uniqBy((stream: Stream) => stream.getId() || uniqueId()) const { categories, countries, subdivisions, cities, regions } = data diff --git a/scripts/commands/playlist/test.ts b/scripts/commands/playlist/test.ts index b4cecf060b..a96a458907 100644 --- a/scripts/commands/playlist/test.ts +++ b/scripts/commands/playlist/test.ts @@ -164,7 +164,7 @@ async function removeBrokenLinks() { streams = streams.filter((stream: Stream) => !isBroken(stream)) - const playlist = new Playlist(streams, { public: false }) + const playlist = new Playlist(streams, { public: false, raw: true }) await rootStorage.save(filepath, playlist.toString()) } } diff --git a/scripts/commands/playlist/update.ts b/scripts/commands/playlist/update.ts index ee04f10773..5fc80007a4 100644 --- a/scripts/commands/playlist/update.ts +++ b/scripts/commands/playlist/update.ts @@ -68,7 +68,7 @@ async function saveStreams() { let filteredStreams = new Collection(groupedStreams.get(filepath)) filteredStreams = filteredStreams.filter((stream: Stream) => stream.removed === false) - const playlist = new Playlist(filteredStreams, { public: false }) + const playlist = new Playlist(filteredStreams, { public: false, raw: true }) await streamsStorage.save(filepath, playlist.toString()) } } diff --git a/scripts/commands/playlist/validate.ts b/scripts/commands/playlist/validate.ts index 30bb066b90..7e5d0599e5 100644 --- a/scripts/commands/playlist/validate.ts +++ b/scripts/commands/playlist/validate.ts @@ -1,4 +1,5 @@ import { Logger, Collection, Dictionary } from '@freearhey/core' +import { OptionValues, program } from 'commander' import { Storage } from '@freearhey/storage-js' import { PlaylistParser } from '../../core' import { data, loadData } from '../../api' @@ -6,10 +7,14 @@ import { ROOT_DIR } from '../../constants' import { isURI } from '../../utils.js' import { Stream } from '../../models' import * as sdk from '@iptv-org/sdk' -import { program } from 'commander' import chalk from 'chalk' -program.argument('[filepath...]', 'Path to file to validate').parse(process.argv) +program + .argument('[filepath...]', 'Path to file to validate') + .option('--log-level ', 'Specify the minimum log level (error or warning)') + .parse(process.argv) + +const options: OptionValues = program.opts() type LogItem = { type: string @@ -51,7 +56,7 @@ async function main() { const streams = streamsGroupedByFilepath.get(filepath) if (!streams) continue - const log = new Collection() + let log = new Collection() streams.forEach((stream: Stream) => { if (!stream.channel) { log.add({ @@ -130,6 +135,12 @@ async function main() { } }) + if (options.logLevel === 'error') { + log = log.filter(i => ['error'].includes(i.type)) + } else if (options.logLevel === 'warning') { + log = log.filter(i => ['warning', 'error'].includes(i.type)) + } + if (log.isNotEmpty()) { console.log(`\n${chalk.underline(filepath)}`) diff --git a/scripts/core/dataSet.ts b/scripts/core/dataSet.ts index d8a838ac52..b8f354424e 100644 --- a/scripts/core/dataSet.ts +++ b/scripts/core/dataSet.ts @@ -10,7 +10,7 @@ export class DataSet { return this._data.has(key) } - missing(key: string): boolean { + missing(key: string | string[]): boolean { const keys = Array.isArray(key) ? key : [key] return keys.every(_key => this._data.get(_key) === undefined) diff --git a/scripts/generators/categoriesGenerator.ts b/scripts/generators/categoriesGenerator.ts index d42719392e..ed1f4d94af 100644 --- a/scripts/generators/categoriesGenerator.ts +++ b/scripts/generators/categoriesGenerator.ts @@ -25,7 +25,7 @@ export class CategoriesGenerator implements Generator { } async generate() { - const streams = this.streams.sortBy([(stream: Stream) => stream.title]) + const streams = this.streams this.categories.forEach(async (category: sdk.Models.Category) => { const categoryStreams = streams @@ -41,7 +41,7 @@ export class CategoriesGenerator implements Generator { return stream }) - const playlist = new Playlist(categoryStreams, { public: true }) + const playlist = new Playlist(categoryStreams, { public: true, raw: false }) const filepath = `categories/${category.id}.m3u` await this.storage.save(filepath, playlist.toString()) this.logFile.append( @@ -50,7 +50,7 @@ export class CategoriesGenerator implements Generator { }) const undefinedStreams = streams.filter((stream: Stream) => stream.getCategories().isEmpty()) - const playlist = new Playlist(undefinedStreams, { public: true }) + const playlist = new Playlist(undefinedStreams, { public: true, raw: false }) const filepath = 'categories/undefined.m3u' await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/citiesGenerator.ts b/scripts/generators/citiesGenerator.ts index 237e8da3dc..e75b8d3dce 100644 --- a/scripts/generators/citiesGenerator.ts +++ b/scripts/generators/citiesGenerator.ts @@ -25,9 +25,7 @@ export class CitiesGenerator implements Generator { } async generate(): Promise { - const streams = this.streams - .sortBy((stream: Stream) => stream.title) - .filter((stream: Stream) => stream.isSFW()) + const streams = this.streams.filter((stream: Stream) => stream.isSFW()) const streamsGroupedByCityCode = {} streams.forEach((stream: Stream) => { @@ -43,7 +41,7 @@ export class CitiesGenerator implements Generator { for (const cityCode in streamsGroupedByCityCode) { const cityStreams = streamsGroupedByCityCode[cityCode] - const playlist = new Playlist(cityStreams, { public: true }) + const playlist = new Playlist(cityStreams, { public: true, raw: false }) const filepath = `cities/${cityCode.toLowerCase()}.m3u` await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/countriesGenerator.ts b/scripts/generators/countriesGenerator.ts index 0ee466dc62..f5a5c09c39 100644 --- a/scripts/generators/countriesGenerator.ts +++ b/scripts/generators/countriesGenerator.ts @@ -25,9 +25,7 @@ export class CountriesGenerator implements Generator { } async generate(): Promise { - const streams = this.streams - .sortBy((stream: Stream) => stream.title) - .filter((stream: Stream) => stream.isSFW()) + const streams = this.streams.filter((stream: Stream) => stream.isSFW()) const streamsGroupedByCountryCode = {} streams.forEach((stream: Stream) => { @@ -43,7 +41,7 @@ export class CountriesGenerator implements Generator { for (const countryCode in streamsGroupedByCountryCode) { const countryStreams = streamsGroupedByCountryCode[countryCode] - const playlist = new Playlist(countryStreams, { public: true }) + const playlist = new Playlist(countryStreams, { public: true, raw: false }) const filepath = `countries/${countryCode.toLowerCase()}.m3u` await this.storage.save(filepath, playlist.toString()) this.logFile.append( @@ -52,7 +50,10 @@ export class CountriesGenerator implements Generator { } const internationalStreams = streams.filter((stream: Stream) => stream.isInternational()) - const internationalPlaylist = new Playlist(internationalStreams, { public: true }) + const internationalPlaylist = new Playlist(internationalStreams, { + public: true, + displayUniqueName: true + }) const internationalFilepath = 'countries/int.m3u' await this.storage.save(internationalFilepath, internationalPlaylist.toString()) this.logFile.append( @@ -66,7 +67,7 @@ export class CountriesGenerator implements Generator { const undefinedStreams = streams.filter((stream: Stream) => stream.getBroadcastAreaCodes().isEmpty() ) - const undefinedPlaylist = new Playlist(undefinedStreams, { public: true }) + const undefinedPlaylist = new Playlist(undefinedStreams, { public: true, raw: false }) const undefinedFilepath = 'countries/undefined.m3u' await this.storage.save(undefinedFilepath, undefinedPlaylist.toString()) this.logFile.append( diff --git a/scripts/generators/indexCategoryGenerator.ts b/scripts/generators/indexCategoryGenerator.ts index 08ebf74be8..8d0510b77c 100644 --- a/scripts/generators/indexCategoryGenerator.ts +++ b/scripts/generators/indexCategoryGenerator.ts @@ -22,7 +22,7 @@ export class IndexCategoryGenerator implements Generator { } async generate(): Promise { - const streams = this.streams.sortBy(stream => stream.title).filter(stream => stream.isSFW()) + const streams = this.streams.filter(stream => stream.isSFW()) let groupedStreams = new Collection() streams.forEach((stream: Stream) => { @@ -46,7 +46,7 @@ export class IndexCategoryGenerator implements Generator { return stream.groupTitle }) - const playlist = new Playlist(groupedStreams, { public: true }) + const playlist = new Playlist(groupedStreams, { public: true, raw: false }) const filepath = 'index.category.m3u' await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/indexCountryGenerator.ts b/scripts/generators/indexCountryGenerator.ts index 078a1f9d6a..865fdc30b8 100644 --- a/scripts/generators/indexCountryGenerator.ts +++ b/scripts/generators/indexCountryGenerator.ts @@ -25,7 +25,6 @@ export class IndexCountryGenerator implements Generator { let groupedStreams = new Collection() this.streams - .sortBy((stream: Stream) => stream.title) .filter((stream: Stream) => stream.isSFW()) .forEach((stream: Stream) => { const broadcastAreaCountries = stream.getBroadcastCountries() @@ -57,7 +56,7 @@ export class IndexCountryGenerator implements Generator { return stream.groupTitle }) - const playlist = new Playlist(groupedStreams, { public: true }) + const playlist = new Playlist(groupedStreams, { public: true, raw: false }) const filepath = 'index.country.m3u' await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/indexGenerator.ts b/scripts/generators/indexGenerator.ts index fdfdde19d3..3a1c9d578b 100644 --- a/scripts/generators/indexGenerator.ts +++ b/scripts/generators/indexGenerator.ts @@ -22,7 +22,6 @@ export class IndexGenerator implements Generator { async generate(): Promise { const sfwStreams = this.streams - .sortBy(stream => stream.title) .filter((stream: Stream) => stream.isSFW()) .map((stream: Stream) => { const groupTitle = stream @@ -35,7 +34,7 @@ export class IndexGenerator implements Generator { return stream }) - const playlist = new Playlist(sfwStreams, { public: true }) + const playlist = new Playlist(sfwStreams, { public: true, raw: false }) const filepath = 'index.m3u' await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/indexLanguageGenerator.ts b/scripts/generators/indexLanguageGenerator.ts index 5e93a46adb..530d163091 100644 --- a/scripts/generators/indexLanguageGenerator.ts +++ b/scripts/generators/indexLanguageGenerator.ts @@ -24,7 +24,6 @@ export class IndexLanguageGenerator implements Generator { async generate(): Promise { let groupedStreams = new Collection() this.streams - .sortBy((stream: Stream) => stream.title) .filter((stream: Stream) => stream.isSFW()) .forEach((stream: Stream) => { const streamLanguages = stream.getLanguages() @@ -47,7 +46,7 @@ export class IndexLanguageGenerator implements Generator { return stream.groupTitle }) - const playlist = new Playlist(groupedStreams, { public: true }) + const playlist = new Playlist(groupedStreams, { public: true, raw: false }) const filepath = 'index.language.m3u' await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/languagesGenerator.ts b/scripts/generators/languagesGenerator.ts index 3e7de3f0d7..58ae20e5e1 100644 --- a/scripts/generators/languagesGenerator.ts +++ b/scripts/generators/languagesGenerator.ts @@ -19,9 +19,7 @@ export class LanguagesGenerator implements Generator { } async generate(): Promise { - const streams: Collection = this.streams - .sortBy((stream: Stream) => stream.title) - .filter((stream: Stream) => stream.isSFW()) + const streams: Collection = this.streams.filter((stream: Stream) => stream.isSFW()) const languages = new Collection() streams.forEach((stream: Stream) => { @@ -39,7 +37,7 @@ export class LanguagesGenerator implements Generator { if (languageStreams.isEmpty()) return - const playlist = new Playlist(languageStreams, { public: true }) + const playlist = new Playlist(languageStreams, { public: true, raw: false }) const filepath = `languages/${language.code}.m3u` await this.storage.save(filepath, playlist.toString()) this.logFile.append( @@ -50,7 +48,7 @@ export class LanguagesGenerator implements Generator { const undefinedStreams = streams.filter((stream: Stream) => stream.getLanguages().isEmpty()) if (undefinedStreams.isEmpty()) return - const playlist = new Playlist(undefinedStreams, { public: true }) + const playlist = new Playlist(undefinedStreams, { public: true, raw: false }) const filepath = 'languages/undefined.m3u' await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/rawGenerator.ts b/scripts/generators/rawGenerator.ts index f16bdf76a7..de844500fc 100644 --- a/scripts/generators/rawGenerator.ts +++ b/scripts/generators/rawGenerator.ts @@ -34,7 +34,7 @@ export class RawGenerator implements Generator { return stream }) - const playlist = new Playlist(streams, { public: true }) + const playlist = new Playlist(streams, { public: true, raw: true }) const filepath = `raw/${filename}` await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/regionsGenerator.ts b/scripts/generators/regionsGenerator.ts index c0d669bed6..8ba7fd5084 100644 --- a/scripts/generators/regionsGenerator.ts +++ b/scripts/generators/regionsGenerator.ts @@ -25,9 +25,7 @@ export class RegionsGenerator implements Generator { } async generate(): Promise { - const streams = this.streams - .sortBy((stream: Stream) => stream.title) - .filter((stream: Stream) => stream.isSFW()) + const streams = this.streams.filter((stream: Stream) => stream.isSFW()) const streamsGroupedByRegionCode = {} streams.forEach((stream: Stream) => { @@ -43,7 +41,7 @@ export class RegionsGenerator implements Generator { for (const regionCode in streamsGroupedByRegionCode) { const regionStreams = streamsGroupedByRegionCode[regionCode] - const playlist = new Playlist(regionStreams, { public: true }) + const playlist = new Playlist(regionStreams, { public: true, raw: false }) const filepath = `regions/${regionCode.toLowerCase()}.m3u` await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/sourcesGenerator.ts b/scripts/generators/sourcesGenerator.ts index d90bc7f742..61c3dc94f4 100644 --- a/scripts/generators/sourcesGenerator.ts +++ b/scripts/generators/sourcesGenerator.ts @@ -38,7 +38,7 @@ export class SourcesGenerator implements Generator { return stream }) - const playlist = new Playlist(streams, { public: true }) + const playlist = new Playlist(streams, { public: true, raw: false }) const filepath = `sources/${filename}` await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/generators/subdivisionsGenerator.ts b/scripts/generators/subdivisionsGenerator.ts index 758f6738f2..728153d116 100644 --- a/scripts/generators/subdivisionsGenerator.ts +++ b/scripts/generators/subdivisionsGenerator.ts @@ -25,9 +25,7 @@ export class SubdivisionsGenerator implements Generator { } async generate(): Promise { - const streams = this.streams - .sortBy((stream: Stream) => stream.title) - .filter((stream: Stream) => stream.isSFW()) + const streams = this.streams.filter((stream: Stream) => stream.isSFW()) const streamsGroupedBySubdivisionCode = {} streams.forEach((stream: Stream) => { @@ -43,7 +41,7 @@ export class SubdivisionsGenerator implements Generator { for (const subdivisionCode in streamsGroupedBySubdivisionCode) { const subdivisionStreams = streamsGroupedBySubdivisionCode[subdivisionCode] - const playlist = new Playlist(subdivisionStreams, { public: true }) + const playlist = new Playlist(subdivisionStreams, { public: true, raw: false }) const filepath = `subdivisions/${subdivisionCode.toLowerCase()}.m3u` await this.storage.save(filepath, playlist.toString()) this.logFile.append( diff --git a/scripts/models/playlist.ts b/scripts/models/playlist.ts index 472ebc1474..d3b37178e8 100644 --- a/scripts/models/playlist.ts +++ b/scripts/models/playlist.ts @@ -2,17 +2,18 @@ import { Collection } from '@freearhey/core' import * as sdk from '@iptv-org/sdk' import { Stream } from '../models' -type PlaylistOptions = { +export type PlaylistOptions = { public?: boolean + raw?: boolean } export class Playlist { streams: Collection - public: boolean + options?: PlaylistOptions constructor(streams: Collection, options?: PlaylistOptions) { this.streams = streams - this.public = options?.public === true + this.options = options } getGuides(): Collection { @@ -59,7 +60,7 @@ export class Playlist { output += '\r\n' this.streams.forEach((stream: Stream) => { - output += stream.toString({ public: this.public }) + '\r\n' + output += stream.toString(this.options) + '\r\n' }) return output diff --git a/scripts/models/stream.ts b/scripts/models/stream.ts index 8cb68010f7..82afe16c95 100644 --- a/scripts/models/stream.ts +++ b/scripts/models/stream.ts @@ -5,6 +5,7 @@ import * as sdk from '@iptv-org/sdk' import { DataSet } from '../core' import { data } from '../api' import path from 'node:path' +import { PlaylistOptions } from './playlist' export class Stream extends sdk.Models.Stream { filepath?: string @@ -16,6 +17,12 @@ export class Stream extends sdk.Models.Stream { guides = new Collection() isNot247?: boolean isGeoBlocked?: boolean + hasMainFeed?: boolean = false + hasUniqueName?: boolean = false + channelName?: string + feedName?: string + countryName?: string + channelUniqueName?: string setGuides(guides?: sdk.Models.Guide[]) { this.guides = new Collection(guides) @@ -113,6 +120,17 @@ export class Stream extends sdk.Models.Stream { stream.isNot247 = labels.includes('Not 24/7') stream.isGeoBlocked = labels.includes('Geo-blocked') + const feed = stream.getFeed() + stream.hasMainFeed = feed?.is_main + stream.feedName = feed?.name + + const channel = stream.getChannel() + stream.channelName = channel?.name + + const country = channel?.getCountry() + stream.countryName = country?.name + stream.channelUniqueName = `${channel?.name} (${country?.name})` + return stream } @@ -316,6 +334,14 @@ export class Stream extends sdk.Models.Stream { return false } + hasChannel(): boolean { + return !!this.channel && !!this.getChannel() + } + + hasFeed(): boolean { + return !!this.feed && !!this.getFeed() + } + hasCategory(category: sdk.Models.Category): boolean { const channel = this.getChannel() @@ -418,8 +444,25 @@ export class Stream extends sdk.Models.Stream { return logo ? logo.url : '' } - getFullTitle(): string { - let title = `${this.title}` + getTitle(options: PlaylistOptions): string { + const raw = options.raw || false + let title = '' + if (raw) { + title += this.title + } else { + if (!this.hasUniqueName) { + title += this.channelUniqueName + } else { + title += this.channelName + } + if (this.feedName && !this.hasMainFeed) title += ' ' + this.feedName + } + + return title + } + + getFullTitle(options: PlaylistOptions): string { + let title = this.getTitle(options) if (this.quality) { title += ` (${this.quality})` @@ -447,8 +490,8 @@ export class Stream extends sdk.Models.Stream { return labels } - toString(options: { public?: boolean } = {}) { - options = { ...{ public: false }, ...options } + toString(options: PlaylistOptions = {}) { + options = { ...{ public: false, raw: false }, ...options } let output = `#EXTINF:-1 tvg-id="${this.getTvgId()}"` @@ -466,7 +509,7 @@ export class Stream extends sdk.Models.Stream { output += ` group-title="${this.groupTitle}"` } - output += `,${this.getFullTitle()}` + output += `,${this.getFullTitle(options)}` if (this.referrer) { output += `\r\n#EXTVLCOPT:http-referrer=${this.referrer}`