From acb19e72eeebc15e29faf1fb84f25706c72478fb Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Thu, 10 Jul 2025 21:13:43 +0300 Subject: [PATCH] Update scripts --- scripts/commands/api/generate.ts | 4 +- scripts/commands/api/load.ts | 1 + scripts/commands/playlist/edit.ts | 15 ++- scripts/commands/playlist/format.ts | 6 +- scripts/commands/playlist/generate.ts | 12 +- scripts/commands/playlist/test.ts | 6 +- scripts/commands/playlist/update.ts | 6 +- scripts/commands/playlist/validate.ts | 4 +- scripts/commands/report/create.ts | 6 +- scripts/core/dataLoader.ts | 3 + scripts/core/dataProcessor.ts | 51 +++++---- scripts/core/issueData.ts | 4 +- scripts/core/issueLoader.ts | 2 +- scripts/core/issueParser.ts | 3 +- scripts/core/playlistParser.ts | 11 +- scripts/generators/categoriesGenerator.ts | 5 +- scripts/generators/countriesGenerator.ts | 2 +- scripts/generators/index.ts | 9 +- scripts/generators/indexCategoryGenerator.ts | 2 +- scripts/generators/indexCountryGenerator.ts | 2 +- scripts/generators/indexGenerator.ts | 8 +- scripts/generators/indexLanguageGenerator.ts | 2 +- scripts/generators/indexNsfwGenerator.ts | 2 +- scripts/generators/indexRegionGenerator.ts | 2 +- scripts/generators/languagesGenerator.ts | 2 +- scripts/generators/regionsGenerator.ts | 2 +- scripts/generators/sourcesGenerator.ts | 44 ++++++++ scripts/models/channel.ts | 46 +++++++- scripts/models/index.ts | 1 + scripts/models/logo.ts | 40 +++++++ scripts/models/stream.ts | 110 ++++++++++++++----- scripts/types/channel.d.ts | 2 - scripts/types/dataLoader.d.ts | 1 + scripts/types/dataProcessor.d.ts | 1 + scripts/types/logo.d.ts | 9 ++ scripts/types/stream.d.ts | 1 + 36 files changed, 342 insertions(+), 85 deletions(-) create mode 100644 scripts/generators/sourcesGenerator.ts create mode 100644 scripts/models/logo.ts create mode 100644 scripts/types/logo.d.ts diff --git a/scripts/commands/api/generate.ts b/scripts/commands/api/generate.ts index f264260f9b..14967e87f5 100644 --- a/scripts/commands/api/generate.ts +++ b/scripts/commands/api/generate.ts @@ -13,13 +13,15 @@ async function main() { const dataStorage = new Storage(DATA_DIR) const dataLoader = new DataLoader({ storage: dataStorage }) const data: DataLoaderData = await dataLoader.load() - const { channelsKeyById, feedsGroupedByChannelId }: DataProcessorData = processor.process(data) + const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData = + processor.process(data) logger.info('loading streams...') const streamsStorage = new Storage(STREAMS_DIR) const parser = new PlaylistParser({ storage: streamsStorage, channelsKeyById, + logosGroupedByStreamId, feedsGroupedByChannelId }) const files = await streamsStorage.list('**/*.m3u') diff --git a/scripts/commands/api/load.ts b/scripts/commands/api/load.ts index 3fdc70043c..e4d89120a2 100644 --- a/scripts/commands/api/load.ts +++ b/scripts/commands/api/load.ts @@ -15,6 +15,7 @@ async function main() { loader.download('regions.json'), loader.download('subdivisions.json'), loader.download('feeds.json'), + loader.download('logos.json'), loader.download('timezones.json'), loader.download('guides.json'), loader.download('streams.json') diff --git a/scripts/commands/playlist/edit.ts b/scripts/commands/playlist/edit.ts index d87590b1f2..035d03c217 100644 --- a/scripts/commands/playlist/edit.ts +++ b/scripts/commands/playlist/edit.ts @@ -49,11 +49,20 @@ export default async function main(filepath: string) { const dataStorage = new Storage(DATA_DIR) const loader = new DataLoader({ storage: dataStorage }) const data: DataLoaderData = await loader.load() - const { channels, channelsKeyById, feedsGroupedByChannelId }: DataProcessorData = - processor.process(data) + const { + channels, + channelsKeyById, + feedsGroupedByChannelId, + logosGroupedByStreamId + }: DataProcessorData = processor.process(data) logger.info('loading streams...') - const parser = new PlaylistParser({ storage, feedsGroupedByChannelId, channelsKeyById }) + const parser = new PlaylistParser({ + storage, + feedsGroupedByChannelId, + logosGroupedByStreamId, + channelsKeyById + }) parsedStreams = await parser.parseFile(filepath) const streamsWithoutId = parsedStreams.filter((stream: Stream) => !stream.id) diff --git a/scripts/commands/playlist/format.ts b/scripts/commands/playlist/format.ts index 43868b73e3..2d5c80aa3e 100644 --- a/scripts/commands/playlist/format.ts +++ b/scripts/commands/playlist/format.ts @@ -16,14 +16,16 @@ async function main() { const dataStorage = new Storage(DATA_DIR) const loader = new DataLoader({ storage: dataStorage }) const data: DataLoaderData = await loader.load() - const { channelsKeyById, feedsGroupedByChannelId }: DataProcessorData = processor.process(data) + const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData = + processor.process(data) logger.info('loading streams...') const streamsStorage = new Storage(STREAMS_DIR) const parser = new PlaylistParser({ storage: streamsStorage, channelsKeyById, - feedsGroupedByChannelId + feedsGroupedByChannelId, + logosGroupedByStreamId }) const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u') let streams = await parser.parse(files) diff --git a/scripts/commands/playlist/generate.ts b/scripts/commands/playlist/generate.ts index b903b5a435..87f2b9ee33 100644 --- a/scripts/commands/playlist/generate.ts +++ b/scripts/commands/playlist/generate.ts @@ -14,7 +14,8 @@ import { CountriesGenerator, LanguagesGenerator, RegionsGenerator, - IndexGenerator + IndexGenerator, + SourcesGenerator } from '../../generators' async function main() { @@ -28,6 +29,7 @@ async function main() { const data: DataLoaderData = await loader.load() const { feedsGroupedByChannelId, + logosGroupedByStreamId, channelsKeyById, categories, countries, @@ -39,15 +41,18 @@ async function main() { const parser = new PlaylistParser({ storage: streamsStorage, feedsGroupedByChannelId, + logosGroupedByStreamId, channelsKeyById }) const files = await streamsStorage.list('**/*.m3u') let streams = await parser.parse(files) const totalStreams = streams.count() + logger.info(`found ${totalStreams} streams`) + + logger.info('filtering streams...') streams = streams.uniqBy((stream: Stream) => stream.hasId() ? stream.getChannelId() + stream.getFeedId() : uniqueId() ) - logger.info(`found ${totalStreams} streams (including ${streams.count()} unique)`) logger.info('sorting streams...') streams = streams.orderBy( @@ -79,6 +84,9 @@ async function main() { logFile }).generate() + logger.info('generating sources/...') + await new SourcesGenerator({ streams, logFile }).generate() + logger.info('generating index.m3u...') await new IndexGenerator({ streams, logFile }).generate() diff --git a/scripts/commands/playlist/test.ts b/scripts/commands/playlist/test.ts index 777c19f8d8..5c307fc45e 100644 --- a/scripts/commands/playlist/test.ts +++ b/scripts/commands/playlist/test.ts @@ -61,14 +61,16 @@ async function main() { const dataStorage = new Storage(DATA_DIR) const loader = new DataLoader({ storage: dataStorage }) const data: DataLoaderData = await loader.load() - const { channelsKeyById, feedsGroupedByChannelId }: DataProcessorData = processor.process(data) + const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData = + processor.process(data) logger.info('loading streams...') const rootStorage = new Storage(ROOT_DIR) const parser = new PlaylistParser({ storage: rootStorage, channelsKeyById, - feedsGroupedByChannelId + feedsGroupedByChannelId, + logosGroupedByStreamId }) const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`) streams = await parser.parse(files) diff --git a/scripts/commands/playlist/update.ts b/scripts/commands/playlist/update.ts index 4393714735..afe6101fc7 100644 --- a/scripts/commands/playlist/update.ts +++ b/scripts/commands/playlist/update.ts @@ -20,13 +20,15 @@ async function main() { const dataStorage = new Storage(DATA_DIR) const dataLoader = new DataLoader({ storage: dataStorage }) const data: DataLoaderData = await dataLoader.load() - const { channelsKeyById, feedsGroupedByChannelId }: DataProcessorData = processor.process(data) + const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData = + processor.process(data) logger.info('loading streams...') const streamsStorage = new Storage(STREAMS_DIR) const parser = new PlaylistParser({ storage: streamsStorage, feedsGroupedByChannelId, + logosGroupedByStreamId, channelsKeyById }) const files = await streamsStorage.list('**/*.m3u') @@ -168,6 +170,7 @@ async function addStreams({ const quality = data.getString('quality') || null const httpUserAgent = data.getString('httpUserAgent') || null const httpReferrer = data.getString('httpReferrer') || null + const directives = data.getArray('directives') || [] const stream = new Stream({ channel: channelId, @@ -176,6 +179,7 @@ async function addStreams({ url: streamUrl, user_agent: httpUserAgent, referrer: httpReferrer, + directives, quality, label }) diff --git a/scripts/commands/playlist/validate.ts b/scripts/commands/playlist/validate.ts index cf77973843..bc5c3b435b 100644 --- a/scripts/commands/playlist/validate.ts +++ b/scripts/commands/playlist/validate.ts @@ -26,6 +26,7 @@ async function main() { const { channelsKeyById, feedsGroupedByChannelId, + logosGroupedByStreamId, blocklistRecordsGroupedByChannelId }: DataProcessorData = processor.process(data) @@ -34,7 +35,8 @@ async function main() { const parser = new PlaylistParser({ storage: rootStorage, channelsKeyById, - feedsGroupedByChannelId + feedsGroupedByChannelId, + logosGroupedByStreamId }) const files = program.args.length ? program.args : await rootStorage.list('streams/**/*.m3u') const streams = await parser.parse(files) diff --git a/scripts/commands/report/create.ts b/scripts/commands/report/create.ts index 61b84501f9..03704ac83b 100644 --- a/scripts/commands/report/create.ts +++ b/scripts/commands/report/create.ts @@ -21,6 +21,7 @@ async function main() { const { channelsKeyById, feedsGroupedByChannelId, + logosGroupedByStreamId, blocklistRecordsGroupedByChannelId }: DataProcessorData = processor.process(data) @@ -29,7 +30,8 @@ async function main() { const parser = new PlaylistParser({ storage: streamsStorage, channelsKeyById, - feedsGroupedByChannelId + feedsGroupedByChannelId, + logosGroupedByStreamId }) const files = await streamsStorage.list('**/*.m3u') const streams = await parser.parse(files) @@ -151,7 +153,7 @@ async function main() { else if (!feedId && streamsGroupedByChannelId.has(channelId)) result.status = 'fulfilled' else { const channelData = channelsKeyById.get(channelId) - if (channelData.length && channelData[0].closed) result.status = 'closed' + if (channelData && channelData.isClosed) result.status = 'closed' } channelSearchRequestsBuffer.set(streamId, true) diff --git a/scripts/core/dataLoader.ts b/scripts/core/dataLoader.ts index 2379edc9ee..f99e51b521 100644 --- a/scripts/core/dataLoader.ts +++ b/scripts/core/dataLoader.ts @@ -47,6 +47,7 @@ export class DataLoader { blocklist, channels, feeds, + logos, timezones, guides, streams @@ -59,6 +60,7 @@ export class DataLoader { this.storage.json('blocklist.json'), this.storage.json('channels.json'), this.storage.json('feeds.json'), + this.storage.json('logos.json'), this.storage.json('timezones.json'), this.storage.json('guides.json'), this.storage.json('streams.json') @@ -73,6 +75,7 @@ export class DataLoader { blocklist, channels, feeds, + logos, timezones, guides, streams diff --git a/scripts/core/dataProcessor.ts b/scripts/core/dataProcessor.ts index 2ea8d28685..55e0dbb4b8 100644 --- a/scripts/core/dataProcessor.ts +++ b/scripts/core/dataProcessor.ts @@ -11,7 +11,8 @@ import { Region, Stream, Guide, - Feed + Feed, + Logo } from '../models' export class DataProcessor { @@ -21,6 +22,9 @@ export class DataProcessor { const categories = new Collection(data.categories).map(data => new Category(data)) const categoriesKeyById = categories.keyBy((category: Category) => category.id) + const languages = new Collection(data.languages).map(data => new Language(data)) + const languagesKeyByCode = languages.keyBy((language: Language) => language.code) + const subdivisions = new Collection(data.subdivisions).map(data => new Subdivision(data)) const subdivisionsKeyByCode = subdivisions.keyBy((subdivision: Subdivision) => subdivision.code) const subdivisionsGroupedByCountryCode = subdivisions.groupBy( @@ -30,20 +34,6 @@ export class DataProcessor { let regions = new Collection(data.regions).map(data => new Region(data)) const regionsKeyByCode = regions.keyBy((region: Region) => region.code) - const blocklistRecords = new Collection(data.blocklist).map(data => new BlocklistRecord(data)) - const blocklistRecordsGroupedByChannelId = blocklistRecords.groupBy( - (blocklistRecord: BlocklistRecord) => blocklistRecord.channelId - ) - - const streams = new Collection(data.streams).map(data => new Stream(data)) - const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId()) - - const guides = new Collection(data.guides).map(data => new Guide(data)) - const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId()) - - const languages = new Collection(data.languages).map(data => new Language(data)) - const languagesKeyByCode = languages.keyBy((language: Language) => language.code) - const countries = new Collection(data.countries).map(data => new Country(data) .withRegions(regions) @@ -52,13 +42,16 @@ export class DataProcessor { ) const countriesKeyByCode = countries.keyBy((country: Country) => country.code) - regions = regions.map((region: Region) => region.withCountries(countriesKeyByCode)) - const timezones = new Collection(data.timezones).map(data => new Timezone(data).withCountries(countriesKeyByCode) ) const timezonesKeyById = timezones.keyBy((timezone: Timezone) => timezone.id) + const blocklistRecords = new Collection(data.blocklist).map(data => new BlocklistRecord(data)) + const blocklistRecordsGroupedByChannelId = blocklistRecords.groupBy( + (blocklistRecord: BlocklistRecord) => blocklistRecord.channelId + ) + let channels = new Collection(data.channels).map(data => new Channel(data) .withCategories(categoriesKeyById) @@ -66,6 +59,7 @@ export class DataProcessor { .withSubdivision(subdivisionsKeyByCode) .withCategories(categoriesKeyById) ) + const channelsKeyById = channels.keyBy((channel: Channel) => channel.id) const feeds = new Collection(data.feeds).map(data => @@ -78,14 +72,32 @@ export class DataProcessor { .withBroadcastSubdivisions(subdivisionsKeyByCode) ) const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId) + const feedsGroupedById = feeds.groupBy((feed: Feed) => feed.id) - channels = channels.map((channel: Channel) => channel.withFeeds(feedsGroupedByChannelId)) + let logos = new Collection(data.logos).map(data => new Logo(data).withFeed(feedsGroupedById)) + const logosGroupedByChannelId = logos.groupBy((logo: Logo) => logo.channelId) + const logosGroupedByStreamId = logos.groupBy((logo: Logo) => logo.getStreamId()) + + const streams = new Collection(data.streams).map(data => + new Stream(data).withLogos(logosGroupedByStreamId) + ) + const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId()) + + const guides = new Collection(data.guides).map(data => new Guide(data)) + const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId()) + + regions = regions.map((region: Region) => region.withCountries(countriesKeyByCode)) + + channels = channels.map((channel: Channel) => + channel.withFeeds(feedsGroupedByChannelId).withLogos(logosGroupedByChannelId) + ) return { blocklistRecordsGroupedByChannelId, subdivisionsGroupedByCountryCode, feedsGroupedByChannelId, guidesGroupedByStreamId, + logosGroupedByStreamId, subdivisionsKeyByCode, countriesKeyByCode, languagesKeyByCode, @@ -104,7 +116,8 @@ export class DataProcessor { regions, streams, guides, - feeds + feeds, + logos } } } diff --git a/scripts/core/issueData.ts b/scripts/core/issueData.ts index 61123f4aa1..e185e1b02f 100644 --- a/scripts/core/issueData.ts +++ b/scripts/core/issueData.ts @@ -24,9 +24,11 @@ export class IssueData { return this._data.get(key) === deleteSymbol ? '' : this._data.get(key) } - getArray(key: string): string[] { + getArray(key: string): string[] | undefined { const deleteSymbol = '~' + if (this._data.missing(key)) return undefined + return this._data.get(key) === deleteSymbol ? [] : this._data.get(key).split('\r\n') } } diff --git a/scripts/core/issueLoader.ts b/scripts/core/issueLoader.ts index 1594eeb371..189e923aa9 100644 --- a/scripts/core/issueLoader.ts +++ b/scripts/core/issueLoader.ts @@ -16,7 +16,7 @@ export class IssueLoader { } let issues: object[] = [] if (TESTING) { - issues = (await import('../../tests/__data__/input/playlist_update/issues.js')).default + issues = (await import('../../tests/__data__/input/issues.js')).default } else { issues = await octokit.paginate(octokit.rest.issues.listForRepo, { owner: OWNER, diff --git a/scripts/core/issueParser.ts b/scripts/core/issueParser.ts index d1874ce33c..2a586c1d6c 100644 --- a/scripts/core/issueParser.ts +++ b/scripts/core/issueParser.ts @@ -16,7 +16,8 @@ const FIELDS = new Dictionary({ 'HTTP Referrer': 'httpReferrer', 'What happened to the stream?': 'reason', Reason: 'reason', - Notes: 'notes' + Notes: 'notes', + Directives: 'directives' }) export class IssueParser { diff --git a/scripts/core/playlistParser.ts b/scripts/core/playlistParser.ts index 37e2896dbe..2086bb568e 100644 --- a/scripts/core/playlistParser.ts +++ b/scripts/core/playlistParser.ts @@ -5,17 +5,25 @@ import { Stream } from '../models' type PlaylistPareserProps = { storage: Storage feedsGroupedByChannelId: Dictionary + logosGroupedByStreamId: Dictionary channelsKeyById: Dictionary } export class PlaylistParser { storage: Storage feedsGroupedByChannelId: Dictionary + logosGroupedByStreamId: Dictionary channelsKeyById: Dictionary - constructor({ storage, feedsGroupedByChannelId, channelsKeyById }: PlaylistPareserProps) { + constructor({ + storage, + feedsGroupedByChannelId, + logosGroupedByStreamId, + channelsKeyById + }: PlaylistPareserProps) { this.storage = storage this.feedsGroupedByChannelId = feedsGroupedByChannelId + this.logosGroupedByStreamId = logosGroupedByStreamId this.channelsKeyById = channelsKeyById } @@ -41,6 +49,7 @@ export class PlaylistParser { .fromPlaylistItem(data) .withFeed(this.feedsGroupedByChannelId) .withChannel(this.channelsKeyById) + .withLogos(this.logosGroupedByStreamId) .setFilepath(filepath) return stream diff --git a/scripts/generators/categoriesGenerator.ts b/scripts/generators/categoriesGenerator.ts index 4752dec642..dc29e51d84 100644 --- a/scripts/generators/categoriesGenerator.ts +++ b/scripts/generators/categoriesGenerator.ts @@ -17,7 +17,7 @@ export class CategoriesGenerator implements Generator { logFile: File constructor({ streams, categories, logFile }: CategoriesGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.categories = categories this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile @@ -30,7 +30,8 @@ export class CategoriesGenerator implements Generator { const categoryStreams = streams .filter((stream: Stream) => stream.hasCategory(category)) .map((stream: Stream) => { - stream.groupTitle = stream.getCategoryNames().join(';') + const groupTitle = stream.getCategoryNames().join(';') + if (groupTitle) stream.groupTitle = groupTitle return stream }) diff --git a/scripts/generators/countriesGenerator.ts b/scripts/generators/countriesGenerator.ts index 7dc707cf91..5f3bb5d76f 100644 --- a/scripts/generators/countriesGenerator.ts +++ b/scripts/generators/countriesGenerator.ts @@ -17,7 +17,7 @@ export class CountriesGenerator implements Generator { logFile: File constructor({ streams, countries, logFile }: CountriesGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.countries = countries this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile diff --git a/scripts/generators/index.ts b/scripts/generators/index.ts index 18b6c8e62c..3223cd21f7 100644 --- a/scripts/generators/index.ts +++ b/scripts/generators/index.ts @@ -1,10 +1,11 @@ export * from './categoriesGenerator' export * from './countriesGenerator' -export * from './languagesGenerator' -export * from './regionsGenerator' -export * from './indexGenerator' -export * from './indexNsfwGenerator' export * from './indexCategoryGenerator' export * from './indexCountryGenerator' +export * from './indexGenerator' export * from './indexLanguageGenerator' +export * from './indexNsfwGenerator' export * from './indexRegionGenerator' +export * from './languagesGenerator' +export * from './regionsGenerator' +export * from './sourcesGenerator' diff --git a/scripts/generators/indexCategoryGenerator.ts b/scripts/generators/indexCategoryGenerator.ts index 665f4cb0cf..ce652d7391 100644 --- a/scripts/generators/indexCategoryGenerator.ts +++ b/scripts/generators/indexCategoryGenerator.ts @@ -15,7 +15,7 @@ export class IndexCategoryGenerator implements Generator { logFile: File constructor({ streams, logFile }: IndexCategoryGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile } diff --git a/scripts/generators/indexCountryGenerator.ts b/scripts/generators/indexCountryGenerator.ts index 82eb335efd..ee9599528b 100644 --- a/scripts/generators/indexCountryGenerator.ts +++ b/scripts/generators/indexCountryGenerator.ts @@ -15,7 +15,7 @@ export class IndexCountryGenerator implements Generator { logFile: File constructor({ streams, logFile }: IndexCountryGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile } diff --git a/scripts/generators/indexGenerator.ts b/scripts/generators/indexGenerator.ts index 5cfa86c666..eef4e38695 100644 --- a/scripts/generators/indexGenerator.ts +++ b/scripts/generators/indexGenerator.ts @@ -15,7 +15,7 @@ export class IndexGenerator implements Generator { logFile: File constructor({ streams, logFile }: IndexGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile } @@ -24,6 +24,12 @@ export class IndexGenerator implements Generator { const sfwStreams = this.streams .orderBy(stream => stream.getTitle()) .filter((stream: Stream) => stream.isSFW()) + .map((stream: Stream) => { + const groupTitle = stream.getCategoryNames().join(';') + if (groupTitle) stream.groupTitle = groupTitle + + return stream + }) const playlist = new Playlist(sfwStreams, { public: true }) const filepath = 'index.m3u' diff --git a/scripts/generators/indexLanguageGenerator.ts b/scripts/generators/indexLanguageGenerator.ts index 3df9f71f2e..ce348ec02b 100644 --- a/scripts/generators/indexLanguageGenerator.ts +++ b/scripts/generators/indexLanguageGenerator.ts @@ -15,7 +15,7 @@ export class IndexLanguageGenerator implements Generator { logFile: File constructor({ streams, logFile }: IndexLanguageGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile } diff --git a/scripts/generators/indexNsfwGenerator.ts b/scripts/generators/indexNsfwGenerator.ts index e1e98375b6..ce48ae559d 100644 --- a/scripts/generators/indexNsfwGenerator.ts +++ b/scripts/generators/indexNsfwGenerator.ts @@ -15,7 +15,7 @@ export class IndexNsfwGenerator implements Generator { logFile: File constructor({ streams, logFile }: IndexNsfwGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile } diff --git a/scripts/generators/indexRegionGenerator.ts b/scripts/generators/indexRegionGenerator.ts index c462fcfceb..c95421de34 100644 --- a/scripts/generators/indexRegionGenerator.ts +++ b/scripts/generators/indexRegionGenerator.ts @@ -17,7 +17,7 @@ export class IndexRegionGenerator implements Generator { logFile: File constructor({ streams, regions, logFile }: IndexRegionGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.regions = regions this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile diff --git a/scripts/generators/languagesGenerator.ts b/scripts/generators/languagesGenerator.ts index f7ae9976e4..933b67c75c 100644 --- a/scripts/generators/languagesGenerator.ts +++ b/scripts/generators/languagesGenerator.ts @@ -12,7 +12,7 @@ export class LanguagesGenerator implements Generator { logFile: File constructor({ streams, logFile }: LanguagesGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile } diff --git a/scripts/generators/regionsGenerator.ts b/scripts/generators/regionsGenerator.ts index 4d649a3517..f5efd393b8 100644 --- a/scripts/generators/regionsGenerator.ts +++ b/scripts/generators/regionsGenerator.ts @@ -17,7 +17,7 @@ export class RegionsGenerator implements Generator { logFile: File constructor({ streams, regions, logFile }: RegionsGeneratorProps) { - this.streams = streams + this.streams = streams.clone() this.regions = regions this.storage = new Storage(PUBLIC_DIR) this.logFile = logFile diff --git a/scripts/generators/sourcesGenerator.ts b/scripts/generators/sourcesGenerator.ts new file mode 100644 index 0000000000..30018815b5 --- /dev/null +++ b/scripts/generators/sourcesGenerator.ts @@ -0,0 +1,44 @@ +import { Collection, Storage, File, type Dictionary } from '@freearhey/core' +import { Stream, Playlist } from '../models' +import { PUBLIC_DIR } from '../constants' +import { Generator } from './generator' +import { EOL } from 'node:os' + +type SourcesGeneratorProps = { + streams: Collection + logFile: File +} + +export class SourcesGenerator implements Generator { + streams: Collection + storage: Storage + logFile: File + + constructor({ streams, logFile }: SourcesGeneratorProps) { + this.streams = streams.clone() + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate() { + const files: Dictionary = this.streams.groupBy((stream: Stream) => stream.getFilename()) + + for (let filename of files.keys()) { + if (!filename) continue + + let streams = new Collection(files.get(filename)) + streams = streams.map((stream: Stream) => { + const groupTitle = stream.getCategoryNames().join(';') + if (groupTitle) stream.groupTitle = groupTitle + + return stream + }) + const playlist = new Playlist(streams, { public: true }) + const filepath = `sources/${filename}` + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'source', filepath, count: playlist.streams.count() }) + EOL + ) + } + } +} diff --git a/scripts/models/channel.ts b/scripts/models/channel.ts index cdc09af0ad..09b0c36adf 100644 --- a/scripts/models/channel.ts +++ b/scripts/models/channel.ts @@ -1,5 +1,5 @@ import { Collection, Dictionary } from '@freearhey/core' -import { Category, Country, Feed, Guide, Stream, Subdivision } from './index' +import { Category, Country, Feed, Guide, Logo, Stream, Subdivision } from './index' import type { ChannelData, ChannelSearchableData, ChannelSerializedData } from '../types/channel' export class Channel { @@ -19,9 +19,10 @@ export class Channel { launched?: string closed?: string replacedBy?: string + isClosed: boolean website?: string - logo: string feeds?: Collection + logos: Collection = new Collection() constructor(data?: ChannelData) { if (!data) return @@ -40,7 +41,7 @@ export class Channel { this.closed = data.closed || undefined this.replacedBy = data.replaced_by || undefined this.website = data.website || undefined - this.logo = data.logo + this.isClosed = !!data.closed || !!data.replaced_by } withSubdivision(subdivisionsKeyByCode: Dictionary): this { @@ -71,6 +72,12 @@ export class Channel { return this } + withLogos(logosGroupedByChannelId: Dictionary): this { + if (this.id) this.logos = new Collection(logosGroupedByChannelId.get(this.id)) + + return this + } + getCountry(): Country | undefined { return this.country } @@ -142,6 +149,35 @@ export class Channel { return this.isNSFW === false } + getLogos(): Collection { + function feed(logo: Logo): number { + if (!logo.feed) return 1 + if (logo.feed.isMain) return 1 + + return 0 + } + + function format(logo: Logo): number { + const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 } + + return logo.format ? levelByFormat[logo.format] : 0 + } + + function size(logo: Logo): number { + return Math.abs(512 - logo.width) + Math.abs(512 - logo.height) + } + + return this.logos.orderBy([feed, format, size], ['desc', 'desc', 'asc'], false) + } + + getLogo(): Logo | undefined { + return this.getLogos().first() + } + + hasLogo(): boolean { + return this.getLogos().notEmpty() + } + getSearchable(): ChannelSearchableData { return { id: this.id, @@ -171,8 +207,7 @@ export class Channel { launched: this.launched, closed: this.closed, replacedBy: this.replacedBy, - website: this.website, - logo: this.logo + website: this.website } } @@ -192,7 +227,6 @@ export class Channel { this.closed = data.closed this.replacedBy = data.replacedBy this.website = data.website - this.logo = data.logo return this } diff --git a/scripts/models/index.ts b/scripts/models/index.ts index db4d6f5fa8..4e11d28b97 100644 --- a/scripts/models/index.ts +++ b/scripts/models/index.ts @@ -7,6 +7,7 @@ export * from './feed' export * from './guide' export * from './issue' export * from './language' +export * from './logo' export * from './playlist' export * from './region' export * from './stream' diff --git a/scripts/models/logo.ts b/scripts/models/logo.ts new file mode 100644 index 0000000000..3cc85fb9da --- /dev/null +++ b/scripts/models/logo.ts @@ -0,0 +1,40 @@ +import { Collection, type Dictionary } from '@freearhey/core' +import type { LogoData } from '../types/logo' +import { type Feed } from './feed' + +export class Logo { + channelId: string + feedId?: string + feed: Feed + tags: Collection + width: number + height: number + format?: string + url: string + + constructor(data?: LogoData) { + if (!data) return + + this.channelId = data.channel + this.feedId = data.feed || undefined + this.tags = new Collection(data.tags) + this.width = data.width + this.height = data.height + this.format = data.format || undefined + this.url = data.url + } + + withFeed(feedsKeyById: Dictionary): this { + if (!this.feedId) return this + + this.feed = feedsKeyById.get(this.feedId) + + return this + } + + getStreamId(): string { + if (!this.feedId) return this.channelId + + return `${this.channelId}@${this.feedId}` + } +} diff --git a/scripts/models/stream.ts b/scripts/models/stream.ts index 692bd303be..b440f86da0 100644 --- a/scripts/models/stream.ts +++ b/scripts/models/stream.ts @@ -1,8 +1,9 @@ -import { Feed, Channel, Category, Region, Subdivision, Country, Language } from './index' +import { Feed, Channel, Category, Region, Subdivision, Country, Language, Logo } from './index' import { URL, Collection, Dictionary } from '@freearhey/core' import type { StreamData } from '../types/stream' import parser from 'iptv-playlist-parser' import { IssueData } from '../core' +import path from 'node:path' export class Stream { name?: string @@ -12,6 +13,7 @@ export class Stream { channel?: Channel feedId?: string feed?: Feed + logos: Collection = new Collection() filepath?: string line?: number label?: string @@ -21,6 +23,7 @@ export class Stream { userAgent?: string groupTitle: string = 'Undefined' removed: boolean = false + directives: Collection = new Collection() constructor(data?: StreamData) { if (!data) return @@ -38,6 +41,7 @@ export class Stream { this.verticalResolution = verticalResolution || undefined this.isInterlaced = isInterlaced || undefined this.label = data.label || undefined + this.directives = new Collection(data.directives) } update(issueData: IssueData): this { @@ -46,7 +50,8 @@ export class Stream { quality: issueData.getString('quality'), httpUserAgent: issueData.getString('httpUserAgent'), httpReferrer: issueData.getString('httpReferrer'), - newStreamUrl: issueData.getString('newStreamUrl') + newStreamUrl: issueData.getString('newStreamUrl'), + directives: issueData.getArray('directives') } if (data.label !== undefined) this.label = data.label @@ -54,11 +59,43 @@ export class Stream { if (data.httpUserAgent !== undefined) this.userAgent = data.httpUserAgent if (data.httpReferrer !== undefined) this.referrer = data.httpReferrer if (data.newStreamUrl !== undefined) this.url = data.newStreamUrl + if (data.directives !== undefined) this.directives = new Collection(data.directives) return this } fromPlaylistItem(data: parser.PlaylistItem): this { + function parseTitle(title: string): { + name: string + label: string + quality: string + } { + const [, label] = title.match(/ \[(.*)\]$/) || [null, ''] + title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '') + const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, ''] + title = title.replace(new RegExp(` \\(${quality}\\)$`), '') + + return { name: title, label, quality } + } + + function parseDirectives(string: string) { + let directives = new Collection() + + if (!string) return directives + + const supportedDirectives = ['#EXTVLCOPT', '#KODIPROP'] + const lines = string.split('\r\n') + const regex = new RegExp(`^${supportedDirectives.join('|')}`, 'i') + + lines.forEach((line: string) => { + if (regex.test(line)) { + directives.add(line.trim()) + } + }) + + return directives + } + if (!data.name) throw new Error('"name" property is required') if (!data.url) throw new Error('"url" property is required') @@ -77,6 +114,7 @@ export class Stream { this.url = data.url this.referrer = data.http.referrer || undefined this.userAgent = data.http['user-agent'] || undefined + this.directives = parseDirectives(data.raw) return this } @@ -99,6 +137,12 @@ export class Stream { return this } + withLogos(logosGroupedByStreamId: Dictionary): this { + if (this.id) this.logos = new Collection(logosGroupedByStreamId.get(this.id)) + + return this + } + setId(id: string): this { this.id = id @@ -130,6 +174,12 @@ export class Stream { return this.line || -1 } + getFilename(): string { + if (!this.filepath) return '' + + return path.basename(this.filepath) + } + setFilepath(filepath: string): this { this.filepath = filepath @@ -294,8 +344,35 @@ export class Stream { return this.feed ? this.feed.isInternational() : false } - getLogo(): string { - return this?.channel?.logo || '' + getLogos(): Collection { + function format(logo: Logo): number { + const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 } + + return logo.format ? levelByFormat[logo.format] : 0 + } + + function size(logo: Logo): number { + return Math.abs(512 - logo.width) + Math.abs(512 - logo.height) + } + + return this.logos.orderBy([format, size], ['desc', 'asc'], false) + } + + getLogo(): Logo | undefined { + return this.getLogos().first() + } + + hasLogo(): boolean { + return this.getLogos().notEmpty() + } + + getLogoUrl(): string { + let logo: Logo | undefined + + if (this.hasLogo()) logo = this.getLogo() + else logo = this?.channel?.getLogo() + + return logo ? logo.url : '' } getName(): string { @@ -339,7 +416,7 @@ export class Stream { let output = `#EXTINF:-1 tvg-id="${this.getId()}"` if (options.public) { - output += ` tvg-logo="${this.getLogo()}" group-title="${this.groupTitle}"` + output += ` tvg-logo="${this.getLogoUrl()}" group-title="${this.groupTitle}"` } if (this.referrer) { @@ -352,13 +429,9 @@ export class Stream { output += `,${this.getTitle()}` - if (this.referrer) { - output += `\r\n#EXTVLCOPT:http-referrer=${this.referrer}` - } - - if (this.userAgent) { - output += `\r\n#EXTVLCOPT:http-user-agent=${this.userAgent}` - } + this.directives.forEach((prop: string) => { + output += `\r\n${prop}` + }) output += `\r\n${this.url}` @@ -366,19 +439,6 @@ export class Stream { } } -function parseTitle(title: string): { - name: string - label: string - quality: string -} { - const [, label] = title.match(/ \[(.*)\]$/) || [null, ''] - title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '') - const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, ''] - title = title.replace(new RegExp(` \\(${quality}\\)$`), '') - - return { name: title, label, quality } -} - function escapeRegExp(text) { return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&') } diff --git a/scripts/types/channel.d.ts b/scripts/types/channel.d.ts index 1f9d031cb1..da36bce184 100644 --- a/scripts/types/channel.d.ts +++ b/scripts/types/channel.d.ts @@ -21,7 +21,6 @@ export type ChannelSerializedData = { closed?: string replacedBy?: string website?: string - logo: string } export type ChannelData = { @@ -39,7 +38,6 @@ export type ChannelData = { closed: string replaced_by: string website: string - logo: string } export type ChannelSearchableData = { diff --git a/scripts/types/dataLoader.d.ts b/scripts/types/dataLoader.d.ts index 05742ff9d1..ee7ab0f937 100644 --- a/scripts/types/dataLoader.d.ts +++ b/scripts/types/dataLoader.d.ts @@ -13,6 +13,7 @@ export type DataLoaderData = { blocklist: object | object[] channels: object | object[] feeds: object | object[] + logos: object | object[] timezones: object | object[] guides: object | object[] streams: object | object[] diff --git a/scripts/types/dataProcessor.d.ts b/scripts/types/dataProcessor.d.ts index 1005ff5b23..25f21d1aac 100644 --- a/scripts/types/dataProcessor.d.ts +++ b/scripts/types/dataProcessor.d.ts @@ -5,6 +5,7 @@ export type DataProcessorData = { subdivisionsGroupedByCountryCode: Dictionary feedsGroupedByChannelId: Dictionary guidesGroupedByStreamId: Dictionary + logosGroupedByStreamId: Dictionary subdivisionsKeyByCode: Dictionary countriesKeyByCode: Dictionary languagesKeyByCode: Dictionary diff --git a/scripts/types/logo.d.ts b/scripts/types/logo.d.ts new file mode 100644 index 0000000000..47cc384537 --- /dev/null +++ b/scripts/types/logo.d.ts @@ -0,0 +1,9 @@ +export type LogoData = { + channel: string + feed: string | null + tags: string[] + width: number + height: number + format: string | null + url: string +} diff --git a/scripts/types/stream.d.ts b/scripts/types/stream.d.ts index 667ad25861..7abb145405 100644 --- a/scripts/types/stream.d.ts +++ b/scripts/types/stream.d.ts @@ -7,4 +7,5 @@ export type StreamData = { user_agent: string | null quality: string | null label: string | null + directives: string[] }