diff --git a/scripts/api.ts b/scripts/api.ts new file mode 100644 index 0000000000..a4649d427f --- /dev/null +++ b/scripts/api.ts @@ -0,0 +1,151 @@ +import { Collection, Dictionary } from '@freearhey/core' +import { DATA_DIR } from './constants' +import cliProgress from 'cli-progress' +import * as sdk from '@iptv-org/sdk' + +const data = { + categoriesKeyById: new Dictionary(), + countriesKeyByCode: new Dictionary(), + subdivisionsKeyByCode: new Dictionary(), + citiesKeyByCode: new Dictionary(), + regionsKeyByCode: new Dictionary(), + languagesKeyByCode: new Dictionary(), + channelsKeyById: new Dictionary(), + feedsKeyByStreamId: new Dictionary(), + feedsGroupedByChannel: new Dictionary(), + blocklistRecordsGroupedByChannel: new Dictionary(), + categories: new Collection(), + countries: new Collection(), + subdivisions: new Collection(), + cities: new Collection(), + regions: new Collection() +} + +let searchIndex + +async function loadData() { + const dataManager = new sdk.DataManager({ dataDir: DATA_DIR }) + await dataManager.loadFromDisk() + dataManager.processData() + + const { + channels, + feeds, + categories, + languages, + countries, + subdivisions, + cities, + regions, + blocklist + } = dataManager.getProcessedData() + + searchIndex = sdk.SearchEngine.createIndex(channels) + + data.categoriesKeyById = categories.keyBy((category: sdk.Models.Category) => category.id) + data.countriesKeyByCode = countries.keyBy((country: sdk.Models.Country) => country.code) + data.subdivisionsKeyByCode = subdivisions.keyBy( + (subdivision: sdk.Models.Subdivision) => subdivision.code + ) + data.citiesKeyByCode = cities.keyBy((city: sdk.Models.City) => city.code) + data.regionsKeyByCode = regions.keyBy((region: sdk.Models.Region) => region.code) + data.languagesKeyByCode = languages.keyBy((language: sdk.Models.Language) => language.code) + data.channelsKeyById = channels.keyBy((channel: sdk.Models.Channel) => channel.id) + data.feedsKeyByStreamId = feeds.keyBy((feed: sdk.Models.Feed) => feed.getStreamId()) + data.feedsGroupedByChannel = feeds.groupBy((feed: sdk.Models.Feed) => feed.channel) + data.blocklistRecordsGroupedByChannel = blocklist.groupBy( + (blocklistRecord: sdk.Models.BlocklistRecord) => blocklistRecord.channel + ) + data.categories = categories + data.countries = countries + data.subdivisions = subdivisions + data.cities = cities + data.regions = regions +} + +async function downloadData() { + function formatBytes(bytes: number) { + if (bytes === 0) return '0 B' + const k = 1024 + const sizes = ['B', 'KB', 'MB', 'GB'] + const i = Math.floor(Math.log(bytes) / Math.log(k)) + return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i] + } + + const files = [ + 'blocklist', + 'categories', + 'channels', + 'cities', + 'countries', + 'feeds', + 'guides', + 'languages', + 'logos', + 'regions', + 'streams', + 'subdivisions', + 'timezones' + ] + + const multiBar = new cliProgress.MultiBar({ + stopOnComplete: true, + hideCursor: true, + forceRedraw: true, + barsize: 36, + format(options, params, payload) { + const filename = payload.filename.padEnd(18, ' ') + const barsize = options.barsize || 40 + const percent = (params.progress * 100).toFixed(2) + const speed = payload.speed ? formatBytes(payload.speed) + '/s' : 'N/A' + const total = formatBytes(params.total) + const completeSize = Math.round(params.progress * barsize) + const incompleteSize = barsize - completeSize + const bar = + options.barCompleteString && options.barIncompleteString + ? options.barCompleteString.substr(0, completeSize) + + options.barGlue + + options.barIncompleteString.substr(0, incompleteSize) + : '-'.repeat(barsize) + + return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}` + } + }) + + const dataManager = new sdk.DataManager({ dataDir: DATA_DIR }) + + const requests: Promise[] = [] + for (const basename of files) { + const filename = `${basename}.json` + const progressBar = multiBar.create(0, 0, { filename }) + const request = dataManager.downloadFileToDisk(basename, { + onDownloadProgress({ total, loaded, rate }) { + if (total) progressBar.setTotal(total) + progressBar.update(loaded, { speed: rate }) + } + }) + + requests.push(request) + } + + await Promise.allSettled(requests).catch(console.error) +} + +function searchChannels(query: string): Collection { + if (!searchIndex) return new Collection() + + const results = searchIndex.search(query) + + const channels = new Collection() + + new Collection(results).forEach( + (item: sdk.Types.ChannelSearchableData) => { + const channel = data.channelsKeyById.get(item.id) + if (channel) channels.add(channel) + } + ) + + return channels +} + +export { data, loadData, downloadData, searchChannels } diff --git a/scripts/commands/api/generate.ts b/scripts/commands/api/generate.ts index 14967e87f5..71c00a6f2e 100644 --- a/scripts/commands/api/generate.ts +++ b/scripts/commands/api/generate.ts @@ -1,39 +1,31 @@ -import { DataLoader, DataProcessor, PlaylistParser } from '../../core' -import type { DataProcessorData } from '../../types/dataProcessor' -import { API_DIR, STREAMS_DIR, DATA_DIR } from '../../constants' -import type { DataLoaderData } from '../../types/dataLoader' -import { Logger, Storage } from '@freearhey/core' -import { Stream } from '../../models' - -async function main() { - const logger = new Logger() - - logger.info('loading data from api...') - const processor = new DataProcessor() - const dataStorage = new Storage(DATA_DIR) - const dataLoader = new DataLoader({ storage: dataStorage }) - const data: DataLoaderData = await dataLoader.load() - const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData = - processor.process(data) - - logger.info('loading streams...') - const streamsStorage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ - storage: streamsStorage, - channelsKeyById, - logosGroupedByStreamId, - feedsGroupedByChannelId - }) - const files = await streamsStorage.list('**/*.m3u') - let streams = await parser.parse(files) - streams = streams - .orderBy((stream: Stream) => stream.getId()) - .map((stream: Stream) => stream.toJSON()) - logger.info(`found ${streams.count()} streams`) - - logger.info('saving to .api/streams.json...') - const apiStorage = new Storage(API_DIR) - await apiStorage.save('streams.json', streams.toJSON()) -} - -main() +import { API_DIR, STREAMS_DIR } from '../../constants' +import { Storage } from '@freearhey/storage-js' +import { PlaylistParser } from '../../core' +import { Logger } from '@freearhey/core' +import { Stream } from '../../models' +import { loadData } from '../../api' + +async function main() { + const logger = new Logger() + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ + storage: streamsStorage + }) + const files = await streamsStorage.list('**/*.m3u') + const parsed = await parser.parse(files) + const _streams = parsed + .sortBy((stream: Stream) => stream.getId()) + .map((stream: Stream) => stream.toObject()) + logger.info(`found ${_streams.count()} streams`) + + logger.info('saving to .api/streams.json...') + const apiStorage = new Storage(API_DIR) + await apiStorage.save('streams.json', _streams.toJSON()) +} + +main() diff --git a/scripts/commands/api/load.ts b/scripts/commands/api/load.ts index 39cf0a2e8b..42919ffd8d 100644 --- a/scripts/commands/api/load.ts +++ b/scripts/commands/api/load.ts @@ -1,26 +1,7 @@ -import { DATA_DIR } from '../../constants' -import { Storage } from '@freearhey/core' -import { DataLoader } from '../../core' - -async function main() { - const storage = new Storage(DATA_DIR) - const loader = new DataLoader({ storage }) - - await Promise.all([ - loader.download('blocklist.json'), - loader.download('categories.json'), - loader.download('channels.json'), - loader.download('countries.json'), - loader.download('languages.json'), - loader.download('regions.json'), - loader.download('subdivisions.json'), - loader.download('feeds.json'), - loader.download('logos.json'), - loader.download('timezones.json'), - loader.download('guides.json'), - loader.download('streams.json'), - loader.download('cities.json') - ]) -} - -main() +import { downloadData } from '../../api' + +async function main() { + await downloadData() +} + +main() diff --git a/scripts/commands/playlist/edit.ts b/scripts/commands/playlist/edit.ts index c58be822a3..d8719cab32 100644 --- a/scripts/commands/playlist/edit.ts +++ b/scripts/commands/playlist/edit.ts @@ -1,217 +1,190 @@ -import { Storage, Collection, Logger, Dictionary } from '@freearhey/core' -import { DataLoader, DataProcessor, PlaylistParser } from '../../core' -import type { ChannelSearchableData } from '../../types/channel' -import { Channel, Feed, Playlist, Stream } from '../../models' -import { DataProcessorData } from '../../types/dataProcessor' -import { DataLoaderData } from '../../types/dataLoader' -import { select, input } from '@inquirer/prompts' -import { DATA_DIR } from '../../constants' -import nodeCleanup from 'node-cleanup' -import sjs from '@freearhey/search-js' -import { Command } from 'commander' -import readline from 'readline' - -type ChoiceValue = { type: string; value?: Feed | Channel } -type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean } - -if (process.platform === 'win32') { - readline - .createInterface({ - input: process.stdin, - output: process.stdout - }) - .on('SIGINT', function () { - process.emit('SIGINT') - }) -} - -const program = new Command() - -program.argument('', 'Path to *.channels.xml file to edit').parse(process.argv) - -const filepath = program.args[0] -const logger = new Logger() -const storage = new Storage() -let parsedStreams = new Collection() - -main(filepath) -nodeCleanup(() => { - save(filepath) -}) - -export default async function main(filepath: string) { - if (!(await storage.exists(filepath))) { - throw new Error(`File "${filepath}" does not exists`) - } - - logger.info('loading data from api...') - const processor = new DataProcessor() - const dataStorage = new Storage(DATA_DIR) - const loader = new DataLoader({ storage: dataStorage }) - const data: DataLoaderData = await loader.load() - const { - channels, - channelsKeyById, - feedsGroupedByChannelId, - logosGroupedByStreamId - }: DataProcessorData = processor.process(data) - - logger.info('loading streams...') - const parser = new PlaylistParser({ - storage, - feedsGroupedByChannelId, - logosGroupedByStreamId, - channelsKeyById - }) - parsedStreams = await parser.parseFile(filepath) - const streamsWithoutId = parsedStreams.filter((stream: Stream) => !stream.id) - - logger.info( - `found ${parsedStreams.count()} streams (including ${streamsWithoutId.count()} without ID)` - ) - - logger.info('creating search index...') - const items = channels.map((channel: Channel) => channel.getSearchable()).all() - const searchIndex = sjs.createIndex(items, { - searchable: ['name', 'altNames', 'guideNames', 'streamTitles', 'feedFullNames'] - }) - - logger.info('starting...\n') - - for (const stream of streamsWithoutId.all()) { - try { - stream.id = await selectChannel(stream, searchIndex, feedsGroupedByChannelId, channelsKeyById) - } catch (err) { - logger.info(err.message) - break - } - } - - streamsWithoutId.forEach((stream: Stream) => { - if (stream.id === '-') { - stream.id = '' - } - }) -} - -async function selectChannel( - stream: Stream, - searchIndex, - feedsGroupedByChannelId: Dictionary, - channelsKeyById: Dictionary -): Promise { - const query = escapeRegex(stream.getTitle()) - const similarChannels = searchIndex - .search(query) - .map((item: ChannelSearchableData) => channelsKeyById.get(item.id)) - - const url = stream.url.length > 50 ? stream.url.slice(0, 50) + '...' : stream.url - - const selected: ChoiceValue = await select({ - message: `Select channel ID for "${stream.title}" (${url}):`, - choices: getChannelChoises(new Collection(similarChannels)), - pageSize: 10 - }) - - switch (selected.type) { - case 'skip': - return '-' - case 'type': { - const typedChannelId = await input({ message: ' Channel ID:' }) - if (!typedChannelId) return '' - const selectedFeedId = await selectFeed(typedChannelId, feedsGroupedByChannelId) - if (selectedFeedId === '-') return typedChannelId - return [typedChannelId, selectedFeedId].join('@') - } - case 'channel': { - const selectedChannel = selected.value - if (!selectedChannel) return '' - const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId) - if (selectedFeedId === '-') return selectedChannel.id - return [selectedChannel.id, selectedFeedId].join('@') - } - } - - return '' -} - -async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary): Promise { - const channelFeeds = new Collection(feedsGroupedByChannelId.get(channelId)) - const choices = getFeedChoises(channelFeeds) - - const selected: ChoiceValue = await select({ - message: `Select feed ID for "${channelId}":`, - choices, - pageSize: 10 - }) - - switch (selected.type) { - case 'skip': - return '-' - case 'type': - return await input({ message: ' Feed ID:', default: 'SD' }) - case 'feed': - const selectedFeed = selected.value - if (!selectedFeed) return '' - return selectedFeed.id - } - - return '' -} - -function getChannelChoises(channels: Collection): Choice[] { - const choises: Choice[] = [] - - channels.forEach((channel: Channel) => { - const names = new Collection([channel.name, ...channel.altNames.all()]).uniq().join(', ') - - choises.push({ - value: { - type: 'channel', - value: channel - }, - name: `${channel.id} (${names})`, - short: `${channel.id}` - }) - }) - - choises.push({ name: 'Type...', value: { type: 'type' } }) - choises.push({ name: 'Skip', value: { type: 'skip' } }) - - return choises -} - -function getFeedChoises(feeds: Collection): Choice[] { - const choises: Choice[] = [] - - feeds.forEach((feed: Feed) => { - let name = `${feed.id} (${feed.name})` - if (feed.isMain) name += ' [main]' - - choises.push({ - value: { - type: 'feed', - value: feed - }, - default: feed.isMain, - name, - short: feed.id - }) - }) - - choises.push({ name: 'Type...', value: { type: 'type' } }) - choises.push({ name: 'Skip', value: { type: 'skip' } }) - - return choises -} - -function save(filepath: string) { - if (!storage.existsSync(filepath)) return - const playlist = new Playlist(parsedStreams) - storage.saveSync(filepath, playlist.toString()) - logger.info(`\nFile '${filepath}' successfully saved`) -} - -function escapeRegex(string: string) { - return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&') -} +import { loadData, data, searchChannels } from '../../api' +import { Collection, Logger } from '@freearhey/core' +import { select, input } from '@inquirer/prompts' +import { Playlist, Stream } from '../../models' +import { Storage } from '@freearhey/storage-js' +import { PlaylistParser } from '../../core' +import nodeCleanup from 'node-cleanup' +import * as sdk from '@iptv-org/sdk' +import { truncate } from '../../utils' +import { Command } from 'commander' +import readline from 'readline' + +type ChoiceValue = { type: string; value?: sdk.Models.Feed | sdk.Models.Channel } +type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean } + +if (process.platform === 'win32') { + readline + .createInterface({ + input: process.stdin, + output: process.stdout + }) + .on('SIGINT', function () { + process.emit('SIGINT') + }) +} + +const program = new Command() + +program.argument('', 'Path to *.channels.xml file to edit').parse(process.argv) + +const filepath = program.args[0] +const logger = new Logger() +const storage = new Storage() +let parsedStreams = new Collection() + +main(filepath) +nodeCleanup(() => { + save(filepath) +}) + +export default async function main(filepath: string) { + if (!(await storage.exists(filepath))) { + throw new Error(`File "${filepath}" does not exists`) + } + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + const parser = new PlaylistParser({ + storage + }) + parsedStreams = await parser.parseFile(filepath) + const streamsWithoutId = parsedStreams.filter((stream: Stream) => !stream.tvgId) + + logger.info( + `found ${parsedStreams.count()} streams (including ${streamsWithoutId.count()} without ID)` + ) + + logger.info('starting...\n') + + for (const stream of streamsWithoutId.all()) { + try { + stream.tvgId = await selectChannel(stream) + } catch (err) { + logger.info(err.message) + break + } + } + + streamsWithoutId.forEach((stream: Stream) => { + if (stream.channel === '-') { + stream.channel = '' + } + }) +} + +async function selectChannel(stream: Stream): Promise { + const query = escapeRegex(stream.title) + const similarChannels = searchChannels(query) + const url = truncate(stream.url, 50) + + const selected: ChoiceValue = await select({ + message: `Select channel ID for "${stream.title}" (${url}):`, + choices: getChannelChoises(similarChannels), + pageSize: 10 + }) + + switch (selected.type) { + case 'skip': + return '-' + case 'type': { + const typedChannelId = await input({ message: ' Channel ID:' }) + if (!typedChannelId) return '' + const selectedFeedId = await selectFeed(typedChannelId) + if (selectedFeedId === '-') return typedChannelId + return [typedChannelId, selectedFeedId].join('@') + } + case 'channel': { + const selectedChannel = selected.value + if (!selectedChannel) return '' + const selectedFeedId = await selectFeed(selectedChannel.id) + if (selectedFeedId === '-') return selectedChannel.id + return [selectedChannel.id, selectedFeedId].join('@') + } + } + + return '' +} + +async function selectFeed(channelId: string): Promise { + const channelFeeds = new Collection(data.feedsGroupedByChannel.get(channelId)) + const choices = getFeedChoises(channelFeeds) + + const selected: ChoiceValue = await select({ + message: `Select feed ID for "${channelId}":`, + choices, + pageSize: 10 + }) + + switch (selected.type) { + case 'skip': + return '-' + case 'type': + return await input({ message: ' Feed ID:', default: 'SD' }) + case 'feed': + const selectedFeed = selected.value + if (!selectedFeed) return '' + return selectedFeed.id + } + + return '' +} + +function getChannelChoises(channels: Collection): Choice[] { + const choises: Choice[] = [] + + channels.forEach((channel: sdk.Models.Channel) => { + const names = new Collection([channel.name, ...channel.alt_names]).uniq().join(', ') + + choises.push({ + value: { + type: 'channel', + value: channel + }, + name: `${channel.id} (${names})`, + short: `${channel.id}` + }) + }) + + choises.push({ name: 'Type...', value: { type: 'type' } }) + choises.push({ name: 'Skip', value: { type: 'skip' } }) + + return choises +} + +function getFeedChoises(feeds: Collection): Choice[] { + const choises: Choice[] = [] + + feeds.forEach((feed: sdk.Models.Feed) => { + let name = `${feed.id} (${feed.name})` + if (feed.is_main) name += ' [main]' + + choises.push({ + value: { + type: 'feed', + value: feed + }, + default: feed.is_main, + name, + short: feed.id + }) + }) + + choises.push({ name: 'Type...', value: { type: 'type' } }) + choises.push({ name: 'Skip', value: { type: 'skip' } }) + + return choises +} + +function save(filepath: string) { + if (!storage.existsSync(filepath)) return + const playlist = new Playlist(parsedStreams) + storage.saveSync(filepath, playlist.toString()) + logger.info(`\nFile '${filepath}' successfully saved`) +} + +function escapeRegex(string: string) { + return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&') +} diff --git a/scripts/commands/playlist/format.ts b/scripts/commands/playlist/format.ts index 345789bc28..15b4bc017e 100644 --- a/scripts/commands/playlist/format.ts +++ b/scripts/commands/playlist/format.ts @@ -1,78 +1,84 @@ -import { Logger, Storage } from '@freearhey/core' -import { STREAMS_DIR, DATA_DIR } from '../../constants' -import { DataLoader, DataProcessor, PlaylistParser } from '../../core' -import { Stream, Playlist } from '../../models' -import { program } from 'commander' -import { DataLoaderData } from '../../types/dataLoader' -import { DataProcessorData } from '../../types/dataProcessor' -import path from 'node:path' - -program.argument('[filepath...]', 'Path to file to format').parse(process.argv) - -async function main() { - const logger = new Logger() - - logger.info('loading data from api...') - const processor = new DataProcessor() - const dataStorage = new Storage(DATA_DIR) - const loader = new DataLoader({ storage: dataStorage }) - const data: DataLoaderData = await loader.load() - const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData = - processor.process(data) - - logger.info('loading streams...') - const streamsStorage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ - storage: streamsStorage, - channelsKeyById, - feedsGroupedByChannelId, - logosGroupedByStreamId - }) - let files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u') - files = files.map((filepath: string) => path.basename(filepath)) - let streams = await parser.parse(files) - - logger.info(`found ${streams.count()} streams`) - - logger.info('normalizing links...') - streams = streams.map(stream => { - stream.normalizeURL() - return stream - }) - - logger.info('removing duplicates...') - streams = streams.uniqBy(stream => stream.url) - - logger.info('removing wrong id...') - streams = streams.map((stream: Stream) => { - if (!stream.channel || channelsKeyById.missing(stream.channel.id)) { - stream.id = '' - } - - return stream - }) - - logger.info('sorting links...') - streams = streams.orderBy( - [ - (stream: Stream) => stream.title, - (stream: Stream) => stream.getVerticalResolution(), - (stream: Stream) => stream.getLabel(), - (stream: Stream) => stream.url - ], - ['asc', 'desc', 'asc', 'asc'] - ) - - logger.info('saving...') - const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath()) - for (const filepath of groupedStreams.keys()) { - const streams = groupedStreams.get(filepath) || [] - - if (!streams.length) return - - const playlist = new Playlist(streams, { public: false }) - await streamsStorage.save(filepath, playlist.toString()) - } -} - -main() +import { Collection, Logger } from '@freearhey/core' +import { Stream, Playlist } from '../../models' +import { Storage } from '@freearhey/storage-js' +import { STREAMS_DIR } from '../../constants' +import { PlaylistParser } from '../../core' +import { loadData } from '../../api' +import { program } from 'commander' +import path from 'node:path' + +program.argument('[filepath...]', 'Path to file to format').parse(process.argv) + +async function main() { + const logger = new Logger() + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ + storage: streamsStorage + }) + let files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u') + files = files.map((filepath: string) => path.basename(filepath)) + let streams = await parser.parse(files) + + logger.info(`found ${streams.count()} streams`) + + logger.info('normalizing links...') + streams = streams.map(stream => { + stream.normalizeURL() + return stream + }) + + logger.info('removing duplicates...') + streams = streams.uniqBy(stream => stream.url) + + logger.info('removing wrong id...') + streams = streams.map((stream: Stream) => { + const channel = stream.getChannel() + if (channel) return stream + + stream.tvgId = '' + stream.channel = '' + stream.feed = '' + + return stream + }) + + logger.info('adding the missing feed id...') + streams = streams.map((stream: Stream) => { + const feed = stream.getFeed() + if (feed) { + stream.feed = feed.id + stream.tvgId = stream.getId() + } + + return stream + }) + + logger.info('sorting links...') + streams = streams.sortBy( + [ + (stream: Stream) => stream.title, + (stream: Stream) => stream.getVerticalResolution(), + (stream: Stream) => stream.label, + (stream: Stream) => stream.url + ], + ['asc', 'desc', 'asc', 'asc'] + ) + + logger.info('saving...') + const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath()) + for (const filepath of groupedStreams.keys()) { + const streams = new Collection(groupedStreams.get(filepath)) + + if (streams.isEmpty()) return + + const playlist = new Playlist(streams, { public: false }) + await streamsStorage.save(filepath, playlist.toString()) + } +} + +main() diff --git a/scripts/commands/playlist/generate.ts b/scripts/commands/playlist/generate.ts index 6e960832a5..a84bd82651 100644 --- a/scripts/commands/playlist/generate.ts +++ b/scripts/commands/playlist/generate.ts @@ -1,131 +1,115 @@ -import { PlaylistParser, DataProcessor, DataLoader } from '../../core' -import type { DataProcessorData } from '../../types/dataProcessor' -import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants' -import type { DataLoaderData } from '../../types/dataLoader' -import { Logger, Storage, File } from '@freearhey/core' -import { Stream } from '../../models' -import uniqueId from 'lodash.uniqueid' -import { - IndexCategoryGenerator, - IndexLanguageGenerator, - IndexCountryGenerator, - SubdivisionsGenerator, - CategoriesGenerator, - CountriesGenerator, - LanguagesGenerator, - RegionsGenerator, - SourcesGenerator, - CitiesGenerator, - IndexGenerator, - RawGenerator -} from '../../generators' - -async function main() { - const logger = new Logger() - const logFile = new File('generators.log') - - logger.info('loading data from api...') - const processor = new DataProcessor() - const dataStorage = new Storage(DATA_DIR) - const loader = new DataLoader({ storage: dataStorage }) - const data: DataLoaderData = await loader.load() - const { - feedsGroupedByChannelId, - logosGroupedByStreamId, - channelsKeyById, - subdivisions, - categories, - countries, - regions, - cities - }: DataProcessorData = processor.process(data) - - logger.info('loading streams...') - const streamsStorage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ - storage: streamsStorage, - feedsGroupedByChannelId, - logosGroupedByStreamId, - channelsKeyById - }) - const files = await streamsStorage.list('**/*.m3u') - let streams = await parser.parse(files) - const totalStreams = streams.count() - logger.info(`found ${totalStreams} streams`) - - logger.info('generating raw/...') - await new RawGenerator({ streams, logFile }).generate() - - logger.info('filtering streams...') - streams = streams.uniqBy((stream: Stream) => - stream.hasId() ? stream.getChannelId() + stream.getFeedId() : uniqueId() - ) - - logger.info('sorting streams...') - streams = streams.orderBy( - [ - (stream: Stream) => stream.getId(), - (stream: Stream) => stream.getVerticalResolution(), - (stream: Stream) => stream.getLabel() - ], - ['asc', 'asc', 'desc'] - ) - - logger.info('generating categories/...') - await new CategoriesGenerator({ categories, streams, logFile }).generate() - - logger.info('generating languages/...') - await new LanguagesGenerator({ streams, logFile }).generate() - - logger.info('generating countries/...') - await new CountriesGenerator({ - countries, - streams, - logFile - }).generate() - - logger.info('generating subdivisions/...') - await new SubdivisionsGenerator({ - subdivisions, - streams, - logFile - }).generate() - - logger.info('generating cities/...') - await new CitiesGenerator({ - cities, - streams, - logFile - }).generate() - - logger.info('generating regions/...') - await new RegionsGenerator({ - streams, - regions, - logFile - }).generate() - - logger.info('generating sources/...') - await new SourcesGenerator({ streams, logFile }).generate() - - logger.info('generating index.m3u...') - await new IndexGenerator({ streams, logFile }).generate() - - logger.info('generating index.category.m3u...') - await new IndexCategoryGenerator({ streams, logFile }).generate() - - logger.info('generating index.country.m3u...') - await new IndexCountryGenerator({ - streams, - logFile - }).generate() - - logger.info('generating index.language.m3u...') - await new IndexLanguageGenerator({ streams, logFile }).generate() - - logger.info('saving generators.log...') - const logStorage = new Storage(LOGS_DIR) - logStorage.saveFile(logFile) -} - -main() +import { LOGS_DIR, STREAMS_DIR } from '../../constants' +import { Storage, File } from '@freearhey/storage-js' +import { PlaylistParser } from '../../core' +import { loadData, data } from '../../api' +import { Logger } from '@freearhey/core' +import uniqueId from 'lodash.uniqueid' +import { Stream } from '../../models' +import { + IndexCategoryGenerator, + IndexLanguageGenerator, + IndexCountryGenerator, + SubdivisionsGenerator, + CategoriesGenerator, + CountriesGenerator, + LanguagesGenerator, + RegionsGenerator, + SourcesGenerator, + CitiesGenerator, + IndexGenerator, + RawGenerator +} from '../../generators' + +async function main() { + const logger = new Logger() + const logFile = new File('generators.log') + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ + storage: streamsStorage + }) + const files = await streamsStorage.list('**/*.m3u') + let streams = await parser.parse(files) + const totalStreams = streams.count() + logger.info(`found ${totalStreams} streams`) + + logger.info('generating raw/...') + await new RawGenerator({ streams, logFile }).generate() + + logger.info('filtering streams...') + streams = streams.uniqBy((stream: Stream) => stream.getId() || uniqueId()) + + logger.info('sorting streams...') + streams = streams.sortBy( + [ + (stream: Stream) => stream.getId(), + (stream: Stream) => stream.getVerticalResolution(), + (stream: Stream) => stream.label + ], + ['asc', 'asc', 'desc'] + ) + + const { categories, countries, subdivisions, cities, regions } = data + + logger.info('generating categories/...') + await new CategoriesGenerator({ categories, streams, logFile }).generate() + + logger.info('generating languages/...') + await new LanguagesGenerator({ streams, logFile }).generate() + + logger.info('generating countries/...') + await new CountriesGenerator({ + countries, + streams, + logFile + }).generate() + + logger.info('generating subdivisions/...') + await new SubdivisionsGenerator({ + subdivisions, + streams, + logFile + }).generate() + + logger.info('generating cities/...') + await new CitiesGenerator({ + cities, + streams, + logFile + }).generate() + + logger.info('generating regions/...') + await new RegionsGenerator({ + streams, + regions, + logFile + }).generate() + + logger.info('generating sources/...') + await new SourcesGenerator({ streams, logFile }).generate() + + logger.info('generating index.m3u...') + await new IndexGenerator({ streams, logFile }).generate() + + logger.info('generating index.category.m3u...') + await new IndexCategoryGenerator({ streams, logFile }).generate() + + logger.info('generating index.country.m3u...') + await new IndexCountryGenerator({ + streams, + logFile + }).generate() + + logger.info('generating index.language.m3u...') + await new IndexLanguageGenerator({ streams, logFile }).generate() + + logger.info('saving generators.log...') + const logStorage = new Storage(LOGS_DIR) + logStorage.saveFile(logFile) +} + +main() diff --git a/scripts/commands/playlist/test.ts b/scripts/commands/playlist/test.ts index f4d59cac1a..737b2dbea7 100644 --- a/scripts/commands/playlist/test.ts +++ b/scripts/commands/playlist/test.ts @@ -1,182 +1,177 @@ -import { Logger, Storage, Collection } from '@freearhey/core' -import { ROOT_DIR, STREAMS_DIR, DATA_DIR } from '../../constants' -import { PlaylistParser, StreamTester, CliTable, DataProcessor, DataLoader } from '../../core' -import type { TestResult } from '../../core/streamTester' -import { Stream } from '../../models' -import { program, OptionValues } from 'commander' -import { eachLimit } from 'async-es' -import chalk from 'chalk' -import os from 'node:os' -import dns from 'node:dns' -import type { DataLoaderData } from '../../types/dataLoader' -import type { DataProcessorData } from '../../types/dataProcessor' - -const LIVE_UPDATE_INTERVAL = 5000 -const LIVE_UPDATE_MAX_STREAMS = 100 - -let errors = 0 -let warnings = 0 -const results: { [key: string]: string } = {} -let interval: string | number | NodeJS.Timeout | undefined -let streams = new Collection() -let isLiveUpdateEnabled = true - -program - .argument('[filepath...]', 'Path to file to test') - .option( - '-p, --parallel ', - 'Batch size of streams to test concurrently', - (value: string) => parseInt(value), - os.cpus().length - ) - .option('-x, --proxy ', 'Use the specified proxy') - .option( - '-t, --timeout ', - 'The number of milliseconds before the request will be aborted', - (value: string) => parseInt(value), - 30000 - ) - .parse(process.argv) - -const options: OptionValues = program.opts() - -const logger = new Logger() -const tester = new StreamTester({ options }) - -async function main() { - if (await isOffline()) { - logger.error(chalk.red('Internet connection is required for the script to work')) - return - } - - logger.info('loading data from api...') - const processor = new DataProcessor() - const dataStorage = new Storage(DATA_DIR) - const loader = new DataLoader({ storage: dataStorage }) - const data: DataLoaderData = await loader.load() - const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData = - processor.process(data) - - logger.info('loading streams...') - const rootStorage = new Storage(ROOT_DIR) - const parser = new PlaylistParser({ - storage: rootStorage, - channelsKeyById, - feedsGroupedByChannelId, - logosGroupedByStreamId - }) - const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`) - streams = await parser.parse(files) - - logger.info(`found ${streams.count()} streams`) - if (streams.count() > LIVE_UPDATE_MAX_STREAMS) isLiveUpdateEnabled = false - - logger.info('starting...') - if (!isLiveUpdateEnabled) { - drawTable() - interval = setInterval(() => { - drawTable() - }, LIVE_UPDATE_INTERVAL) - } - - await eachLimit( - streams.all(), - options.parallel, - async (stream: Stream) => { - await runTest(stream) - - if (isLiveUpdateEnabled) { - drawTable() - } - }, - onFinish - ) -} - -main() - -async function runTest(stream: Stream) { - const key = stream.filepath + stream.getId() + stream.url - results[key] = chalk.white('LOADING...') - - const result: TestResult = await tester.test(stream) - - let status = '' - const errorStatusCodes = ['ENOTFOUND', 'HTTP_404_NOT_FOUND'] - if (result.status.ok) status = chalk.green('OK') - else if (errorStatusCodes.includes(result.status.code)) { - status = chalk.red(result.status.code) - errors++ - } else { - status = chalk.yellow(result.status.code) - warnings++ - } - - results[key] = status -} - -function drawTable() { - process.stdout.write('\u001b[3J\u001b[1J') - console.clear() - - const streamsGrouped = streams.groupBy((stream: Stream) => stream.filepath) - for (const filepath of streamsGrouped.keys()) { - const streams: Stream[] = streamsGrouped.get(filepath) - - const table = new CliTable({ - columns: [ - { name: '', alignment: 'center', minLen: 3, maxLen: 3 }, - { name: 'tvg-id', alignment: 'left', color: 'green', minLen: 25, maxLen: 25 }, - { name: 'url', alignment: 'left', color: 'green', minLen: 100, maxLen: 100 }, - { name: 'status', alignment: 'left', minLen: 25, maxLen: 25 } - ] - }) - streams.forEach((stream: Stream, index: number) => { - const status = results[stream.filepath + stream.getId() + stream.url] || chalk.gray('PENDING') - - const row = { - '': index, - 'tvg-id': stream.getId().length > 25 ? stream.getId().slice(0, 22) + '...' : stream.getId(), - url: stream.url.length > 100 ? stream.url.slice(0, 97) + '...' : stream.url, - status - } - table.append(row) - }) - - process.stdout.write(`\n${chalk.underline(filepath)}\n`) - - process.stdout.write(table.toString()) - } -} - -function onFinish(error: any) { - clearInterval(interval) - - if (error) { - console.error(error) - process.exit(1) - } - - drawTable() - - if (errors > 0 || warnings > 0) { - console.log( - chalk.red(`\n${errors + warnings} problems (${errors} errors, ${warnings} warnings)`) - ) - - if (errors > 0) { - process.exit(1) - } - } - - process.exit(0) -} - -async function isOffline() { - return new Promise((resolve, reject) => { - dns.lookup('info.cern.ch', err => { - if (err) resolve(true) - reject(false) - }) - }).catch(() => {}) -} +import { PlaylistParser, StreamTester, CliTable } from '../../core' +import type { TestResult } from '../../core/streamTester' +import { ROOT_DIR, STREAMS_DIR } from '../../constants' +import { Logger, Collection } from '@freearhey/core' +import { program, OptionValues } from 'commander' +import { Storage } from '@freearhey/storage-js' +import { Stream } from '../../models' +import { loadData } from '../../api' +import { eachLimit } from 'async' +import dns from 'node:dns' +import chalk from 'chalk' +import os from 'node:os' +import { truncate } from '../../utils' + +const LIVE_UPDATE_INTERVAL = 5000 +const LIVE_UPDATE_MAX_STREAMS = 100 + +let errors = 0 +let warnings = 0 +const results: { [key: string]: string } = {} +let interval: string | number | NodeJS.Timeout | undefined +let streams = new Collection() +let isLiveUpdateEnabled = true + +program + .argument('[filepath...]', 'Path to file to test') + .option( + '-p, --parallel ', + 'Batch size of streams to test concurrently', + (value: string) => parseInt(value), + os.cpus().length + ) + .option('-x, --proxy ', 'Use the specified proxy') + .option( + '-t, --timeout ', + 'The number of milliseconds before the request will be aborted', + (value: string) => parseInt(value), + 30000 + ) + .parse(process.argv) + +const options: OptionValues = program.opts() + +const logger = new Logger() +const tester = new StreamTester({ options }) + +async function main() { + if (await isOffline()) { + logger.error(chalk.red('Internet connection is required for the script to work')) + return + } + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + const rootStorage = new Storage(ROOT_DIR) + const parser = new PlaylistParser({ + storage: rootStorage + }) + const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`) + streams = await parser.parse(files) + + logger.info(`found ${streams.count()} streams`) + if (streams.count() > LIVE_UPDATE_MAX_STREAMS) isLiveUpdateEnabled = false + + logger.info('starting...') + if (!isLiveUpdateEnabled) { + drawTable() + interval = setInterval(() => { + drawTable() + }, LIVE_UPDATE_INTERVAL) + } + + eachLimit( + streams.all(), + options.parallel, + async (stream: Stream) => { + await runTest(stream) + + if (isLiveUpdateEnabled) { + drawTable() + } + }, + onFinish + ) +} + +main() + +async function runTest(stream: Stream) { + const key = stream.getUniqKey() + results[key] = chalk.white('LOADING...') + + const result: TestResult = await tester.test(stream) + + let status = '' + const errorStatusCodes = ['ENOTFOUND', 'HTTP_404_NOT_FOUND'] + if (result.status.ok) status = chalk.green('OK') + else if (errorStatusCodes.includes(result.status.code)) { + status = chalk.red(result.status.code) + errors++ + } else { + status = chalk.yellow(result.status.code) + warnings++ + } + + results[key] = status +} + +function drawTable() { + process.stdout.write('\u001b[3J\u001b[1J') + console.clear() + + const streamsGrouped = streams.groupBy((stream: Stream) => stream.filepath) + for (const filepath of streamsGrouped.keys()) { + const streams: Stream[] = streamsGrouped.get(filepath) || [] + + const table = new CliTable({ + columns: [ + { name: '', alignment: 'center', minLen: 3, maxLen: 3 }, + { name: 'tvg-id', alignment: 'left', color: 'green', minLen: 25, maxLen: 25 }, + { name: 'url', alignment: 'left', color: 'green', minLen: 100, maxLen: 100 }, + { name: 'status', alignment: 'left', minLen: 25, maxLen: 25 } + ] + }) + streams.forEach((stream: Stream, index: number) => { + const key = stream.getUniqKey() + const status = results[key] || chalk.gray('PENDING') + const tvgId = stream.getTvgId() + + const row = { + '': index, + 'tvg-id': truncate(tvgId, 25), + url: truncate(stream.url, 100), + status + } + table.append(row) + }) + + process.stdout.write(`\n${chalk.underline(filepath)}\n`) + + process.stdout.write(table.toString()) + } +} + +function onFinish(error: Error) { + clearInterval(interval) + + if (error) { + console.error(error) + process.exit(1) + } + + drawTable() + + if (errors > 0 || warnings > 0) { + console.log( + chalk.red(`\n${errors + warnings} problems (${errors} errors, ${warnings} warnings)`) + ) + + if (errors > 0) { + process.exit(1) + } + } + + process.exit(0) +} + +async function isOffline() { + return new Promise((resolve, reject) => { + dns.lookup('info.cern.ch', err => { + if (err) resolve(true) + reject(false) + }) + }).catch(() => {}) +} diff --git a/scripts/commands/playlist/update.ts b/scripts/commands/playlist/update.ts index 2ded669e75..600b23ac34 100644 --- a/scripts/commands/playlist/update.ts +++ b/scripts/commands/playlist/update.ts @@ -1,194 +1,174 @@ -import { DataLoader, DataProcessor, IssueLoader, PlaylistParser } from '../../core' -import { Logger, Storage, Collection, Dictionary } from '@freearhey/core' -import type { DataProcessorData } from '../../types/dataProcessor' -import { Stream, Playlist, Channel, Issue } from '../../models' -import type { DataLoaderData } from '../../types/dataLoader' -import { DATA_DIR, STREAMS_DIR } from '../../constants' -import { isURI } from '../../utils' - -const processedIssues = new Collection() - -async function main() { - const logger = new Logger({ level: -999 }) - const issueLoader = new IssueLoader() - - logger.info('loading issues...') - const issues = await issueLoader.load() - - logger.info('loading data from api...') - const processor = new DataProcessor() - const dataStorage = new Storage(DATA_DIR) - const dataLoader = new DataLoader({ storage: dataStorage }) - const data: DataLoaderData = await dataLoader.load() - const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData = - processor.process(data) - - logger.info('loading streams...') - const streamsStorage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ - storage: streamsStorage, - feedsGroupedByChannelId, - logosGroupedByStreamId, - channelsKeyById - }) - const files = await streamsStorage.list('**/*.m3u') - const streams = await parser.parse(files) - - logger.info('removing streams...') - await removeStreams({ streams, issues }) - - logger.info('edit stream description...') - await editStreams({ - streams, - issues, - channelsKeyById, - feedsGroupedByChannelId - }) - - logger.info('add new streams...') - await addStreams({ - streams, - issues, - channelsKeyById, - feedsGroupedByChannelId - }) - - logger.info('saving...') - const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath()) - for (const filepath of groupedStreams.keys()) { - let streams = groupedStreams.get(filepath) || [] - streams = streams.filter((stream: Stream) => stream.removed === false) - - const playlist = new Playlist(streams, { public: false }) - await streamsStorage.save(filepath, playlist.toString()) - } - - const output = processedIssues.map(issue_number => `closes #${issue_number}`).join(', ') - console.log(`OUTPUT=${output}`) -} - -main() - -async function removeStreams({ streams, issues }: { streams: Collection; issues: Collection }) { - const requests = issues.filter( - issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved') - ) - requests.forEach((issue: Issue) => { - const data = issue.data - if (data.missing('streamUrl')) return - - const streamUrls = data.getString('streamUrl') || '' - - let changed = false - streamUrls - .split(/\r?\n/) - .filter(Boolean) - .forEach(link => { - const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim()) - if (found) { - found.removed = true - changed = true - } - }) - - if (changed) processedIssues.add(issue.number) - }) -} - -async function editStreams({ - streams, - issues, - channelsKeyById, - feedsGroupedByChannelId -}: { - streams: Collection - issues: Collection - channelsKeyById: Dictionary - feedsGroupedByChannelId: Dictionary -}) { - const requests = issues.filter( - issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved') - ) - requests.forEach((issue: Issue) => { - const data = issue.data - - if (data.missing('streamUrl')) return - - const stream: Stream = streams.first( - (_stream: Stream) => _stream.url === data.getString('streamUrl') - ) - if (!stream) return - - const streamId = data.getString('streamId') || '' - const [channelId, feedId] = streamId.split('@') - - if (channelId) { - stream - .setChannelId(channelId) - .setFeedId(feedId) - .withChannel(channelsKeyById) - .withFeed(feedsGroupedByChannelId) - .updateId() - .updateTitle() - .updateFilepath() - } - - stream.update(data) - - processedIssues.add(issue.number) - }) -} - -async function addStreams({ - streams, - issues, - channelsKeyById, - feedsGroupedByChannelId -}: { - streams: Collection - issues: Collection - channelsKeyById: Dictionary - feedsGroupedByChannelId: Dictionary -}) { - const requests = issues.filter( - issue => issue.labels.includes('streams:add') && issue.labels.includes('approved') - ) - requests.forEach((issue: Issue) => { - const data = issue.data - if (data.missing('streamId') || data.missing('streamUrl')) return - if (streams.includes((_stream: Stream) => _stream.url === data.getString('streamUrl'))) return - const streamUrl = data.getString('streamUrl') || '' - if (!isURI(streamUrl)) return - - const streamId = data.getString('streamId') || '' - const [channelId, feedId] = streamId.split('@') - - const channel: Channel = channelsKeyById.get(channelId) - if (!channel) return - - const label = data.getString('label') || null - const quality = data.getString('quality') || null - const httpUserAgent = data.getString('httpUserAgent') || null - const httpReferrer = data.getString('httpReferrer') || null - const directives = data.getArray('directives') || [] - - const stream = new Stream({ - channelId, - feedId, - title: channel.name, - url: streamUrl, - userAgent: httpUserAgent, - referrer: httpReferrer, - directives, - quality, - label - }) - .withChannel(channelsKeyById) - .withFeed(feedsGroupedByChannelId) - .updateTitle() - .updateFilepath() - - streams.add(stream) - processedIssues.add(issue.number) - }) -} +import { IssueLoader, PlaylistParser } from '../../core' +import { Playlist, Issue, Stream } from '../../models' +import { loadData, data as apiData } from '../../api' +import { Logger, Collection } from '@freearhey/core' +import { Storage } from '@freearhey/storage-js' +import { STREAMS_DIR } from '../../constants' +import * as sdk from '@iptv-org/sdk' +import { isURI } from '../../utils' + +const processedIssues = new Collection() + +async function main() { + const logger = new Logger({ level: -999 }) + const issueLoader = new IssueLoader() + + logger.info('loading issues...') + const issues = await issueLoader.load() + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ + storage: streamsStorage + }) + const files = await streamsStorage.list('**/*.m3u') + const streams = await parser.parse(files) + + logger.info('removing streams...') + await removeStreams({ streams, issues }) + + logger.info('edit stream description...') + await editStreams({ + streams, + issues + }) + + logger.info('add new streams...') + await addStreams({ + streams, + issues + }) + + logger.info('saving...') + const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath()) + for (const filepath of groupedStreams.keys()) { + let streams = new Collection(groupedStreams.get(filepath)) + streams = streams.filter((stream: Stream) => stream.removed === false) + + const playlist = new Playlist(streams, { public: false }) + await streamsStorage.save(filepath, playlist.toString()) + } + + const output = processedIssues.map(issue_number => `closes #${issue_number}`).join(', ') + console.log(`OUTPUT=${output}`) +} + +main() + +async function removeStreams({ + streams, + issues +}: { + streams: Collection + issues: Collection +}) { + const requests = issues.filter( + issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved') + ) + + requests.forEach((issue: Issue) => { + const data = issue.data + if (data.missing('streamUrl')) return + + const streamUrls = data.getString('streamUrl') || '' + + let changed = false + streamUrls + .split(/\r?\n/) + .filter(Boolean) + .forEach(link => { + const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim()) + if (found) { + found.removed = true + changed = true + } + }) + + if (changed) processedIssues.add(issue.number) + }) +} + +async function editStreams({ + streams, + issues +}: { + streams: Collection + issues: Collection +}) { + const requests = issues.filter( + issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved') + ) + requests.forEach((issue: Issue) => { + const data = issue.data + + if (data.missing('streamUrl')) return + + const stream: Stream = streams.first( + (_stream: Stream) => _stream.url === data.getString('streamUrl') + ) + if (!stream) return + + const streamId = data.getString('streamId') || '' + const [channelId, feedId] = streamId.split('@') + + if (channelId) { + stream.channel = channelId + stream.feed = feedId + stream.updateTvgId().updateTitle().updateFilepath() + } + + stream.updateWithIssue(data) + + processedIssues.add(issue.number) + }) +} + +async function addStreams({ + streams, + issues +}: { + streams: Collection + issues: Collection +}) { + const requests = issues.filter( + issue => issue.labels.includes('streams:add') && issue.labels.includes('approved') + ) + requests.forEach((issue: Issue) => { + const data = issue.data + if (data.missing('streamId') || data.missing('streamUrl')) return + if (streams.includes((_stream: Stream) => _stream.url === data.getString('streamUrl'))) return + const streamUrl = data.getString('streamUrl') || '' + if (!isURI(streamUrl)) return + + const streamId = data.getString('streamId') || '' + const [channelId, feedId] = streamId.split('@') + + const channel: sdk.Models.Channel | undefined = apiData.channelsKeyById.get(channelId) + if (!channel) return + + const label = data.getString('label') || '' + const quality = data.getString('quality') || null + const httpUserAgent = data.getString('httpUserAgent') || null + const httpReferrer = data.getString('httpReferrer') || null + const directives = data.getArray('directives') || [] + + const stream = new Stream({ + channel: channelId, + feed: feedId, + title: channel.name, + url: streamUrl, + user_agent: httpUserAgent, + referrer: httpReferrer, + quality + }) + + stream.label = label + stream.setDirectives(directives).updateTitle().updateFilepath() + + streams.add(stream) + processedIssues.add(issue.number) + }) +} diff --git a/scripts/commands/playlist/validate.ts b/scripts/commands/playlist/validate.ts index 823acec73f..36ec037e23 100644 --- a/scripts/commands/playlist/validate.ts +++ b/scripts/commands/playlist/validate.ts @@ -1,129 +1,120 @@ -import { Logger, Storage, Collection, Dictionary } from '@freearhey/core' -import { DataLoader, DataProcessor, PlaylistParser } from '../../core' -import { DataProcessorData } from '../../types/dataProcessor' -import { DATA_DIR, ROOT_DIR } from '../../constants' -import { DataLoaderData } from '../../types/dataLoader' -import { BlocklistRecord, Stream } from '../../models' -import { program } from 'commander' -import chalk from 'chalk' - -program.argument('[filepath...]', 'Path to file to validate').parse(process.argv) - -type LogItem = { - type: string - line: number - message: string -} - -async function main() { - const logger = new Logger() - - logger.info('loading data from api...') - const processor = new DataProcessor() - const dataStorage = new Storage(DATA_DIR) - const loader = new DataLoader({ storage: dataStorage }) - const data: DataLoaderData = await loader.load() - const { - channelsKeyById, - feedsGroupedByChannelId, - logosGroupedByStreamId, - blocklistRecordsGroupedByChannelId - }: DataProcessorData = processor.process(data) - - logger.info('loading streams...') - const rootStorage = new Storage(ROOT_DIR) - const parser = new PlaylistParser({ - storage: rootStorage, - channelsKeyById, - feedsGroupedByChannelId, - logosGroupedByStreamId - }) - const files = program.args.length ? program.args : await rootStorage.list('streams/**/*.m3u') - const streams = await parser.parse(files) - logger.info(`found ${streams.count()} streams`) - - let errors = new Collection() - let warnings = new Collection() - const streamsGroupedByFilepath = streams.groupBy((stream: Stream) => stream.getFilepath()) - for (const filepath of streamsGroupedByFilepath.keys()) { - const streams = streamsGroupedByFilepath.get(filepath) - if (!streams) continue - - const log = new Collection() - const buffer = new Dictionary() - streams.forEach((stream: Stream) => { - if (stream.channelId) { - const channel = channelsKeyById.get(stream.channelId) - if (!channel) { - log.add({ - type: 'warning', - line: stream.getLine(), - message: `"${stream.id}" is not in the database` - }) - } - } - - const duplicate = stream.url && buffer.has(stream.url) - if (duplicate) { - log.add({ - type: 'warning', - line: stream.getLine(), - message: `"${stream.url}" is already on the playlist` - }) - } else { - buffer.set(stream.url, true) - } - - const blocklistRecords = stream.channel - ? new Collection(blocklistRecordsGroupedByChannelId.get(stream.channel.id)) - : new Collection() - - blocklistRecords.forEach((blocklistRecord: BlocklistRecord) => { - if (blocklistRecord.reason === 'dmca') { - log.add({ - type: 'error', - line: stream.getLine(), - message: `"${blocklistRecord.channelId}" is on the blocklist due to claims of copyright holders (${blocklistRecord.ref})` - }) - } else if (blocklistRecord.reason === 'nsfw') { - log.add({ - type: 'error', - line: stream.getLine(), - message: `"${blocklistRecord.channelId}" is on the blocklist due to NSFW content (${blocklistRecord.ref})` - }) - } - }) - }) - - if (log.notEmpty()) { - console.log(`\n${chalk.underline(filepath)}`) - - log.forEach((logItem: LogItem) => { - const position = logItem.line.toString().padEnd(6, ' ') - const type = logItem.type.padEnd(9, ' ') - const status = logItem.type === 'error' ? chalk.red(type) : chalk.yellow(type) - - console.log(` ${chalk.gray(position)}${status}${logItem.message}`) - }) - - errors = errors.concat(log.filter((logItem: LogItem) => logItem.type === 'error')) - warnings = warnings.concat(log.filter((logItem: LogItem) => logItem.type === 'warning')) - } - } - - if (errors.count() || warnings.count()) { - console.log( - chalk.red( - `\n${ - errors.count() + warnings.count() - } problems (${errors.count()} errors, ${warnings.count()} warnings)` - ) - ) - - if (errors.count()) { - process.exit(1) - } - } -} - -main() +import { Logger, Collection, Dictionary } from '@freearhey/core' +import { Storage } from '@freearhey/storage-js' +import { PlaylistParser } from '../../core' +import { data, loadData } from '../../api' +import { ROOT_DIR } from '../../constants' +import { Stream } from '../../models' +import * as sdk from '@iptv-org/sdk' +import { program } from 'commander' +import chalk from 'chalk' + +program.argument('[filepath...]', 'Path to file to validate').parse(process.argv) + +type LogItem = { + type: string + line: number + message: string +} + +async function main() { + const logger = new Logger() + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + const rootStorage = new Storage(ROOT_DIR) + const parser = new PlaylistParser({ + storage: rootStorage + }) + const files = program.args.length ? program.args : await rootStorage.list('streams/**/*.m3u') + const streams = await parser.parse(files) + logger.info(`found ${streams.count()} streams`) + + let errors = new Collection() + let warnings = new Collection() + const streamsGroupedByFilepath = streams.groupBy((stream: Stream) => stream.getFilepath()) + for (const filepath of streamsGroupedByFilepath.keys()) { + const streams = streamsGroupedByFilepath.get(filepath) + if (!streams) continue + + const log = new Collection() + const buffer = new Dictionary() + streams.forEach((stream: Stream) => { + if (stream.channel) { + const channel = data.channelsKeyById.get(stream.channel) + if (!channel) { + log.add({ + type: 'warning', + line: stream.getLine(), + message: `"${stream.tvgId}" is not in the database` + }) + } + } + + const duplicate = stream.url && buffer.has(stream.url) + if (duplicate) { + log.add({ + type: 'warning', + line: stream.getLine(), + message: `"${stream.url}" is already on the playlist` + }) + } else { + buffer.set(stream.url, true) + } + + if (stream.channel) { + const blocklistRecords = new Collection( + data.blocklistRecordsGroupedByChannel.get(stream.channel) + ) + + blocklistRecords.forEach((blocklistRecord: sdk.Models.BlocklistRecord) => { + if (blocklistRecord.reason === 'dmca') { + log.add({ + type: 'error', + line: stream.getLine(), + message: `"${blocklistRecord.channel}" is on the blocklist due to claims of copyright holders (${blocklistRecord.ref})` + }) + } else if (blocklistRecord.reason === 'nsfw') { + log.add({ + type: 'error', + line: stream.getLine(), + message: `"${blocklistRecord.channel}" is on the blocklist due to NSFW content (${blocklistRecord.ref})` + }) + } + }) + } + }) + + if (log.isNotEmpty()) { + console.log(`\n${chalk.underline(filepath)}`) + + log.forEach((logItem: LogItem) => { + const position = logItem.line.toString().padEnd(6, ' ') + const type = logItem.type.padEnd(9, ' ') + const status = logItem.type === 'error' ? chalk.red(type) : chalk.yellow(type) + + console.log(` ${chalk.gray(position)}${status}${logItem.message}`) + }) + + errors = errors.concat(log.filter((logItem: LogItem) => logItem.type === 'error')) + warnings = warnings.concat(log.filter((logItem: LogItem) => logItem.type === 'warning')) + } + } + + if (errors.count() || warnings.count()) { + console.log( + chalk.red( + `\n${ + errors.count() + warnings.count() + } problems (${errors.count()} errors, ${warnings.count()} warnings)` + ) + ) + + if (errors.count()) { + process.exit(1) + } + } +} + +main() diff --git a/scripts/commands/readme/update.ts b/scripts/commands/readme/update.ts index d47f6ba48a..24ce044ab8 100644 --- a/scripts/commands/readme/update.ts +++ b/scripts/commands/readme/update.ts @@ -1,48 +1,30 @@ -import { CategoriesTable, CountriesTable, LanguagesTable, RegionsTable } from '../../tables' -import { DataLoader, DataProcessor, Markdown } from '../../core' -import { DataProcessorData } from '../../types/dataProcessor' -import { DataLoaderData } from '../../types/dataLoader' -import { README_DIR, DATA_DIR, ROOT_DIR } from '../../constants' -import { Logger, Storage } from '@freearhey/core' - -async function main() { - const logger = new Logger() - const dataStorage = new Storage(DATA_DIR) - const processor = new DataProcessor() - const loader = new DataLoader({ storage: dataStorage }) - const data: DataLoaderData = await loader.load() - const { - subdivisionsKeyByCode, - languagesKeyByCode, - countriesKeyByCode, - categoriesKeyById, - subdivisions, - countries, - regions, - cities - }: DataProcessorData = processor.process(data) - - logger.info('creating category table...') - await new CategoriesTable({ categoriesKeyById }).make() - logger.info('creating language table...') - await new LanguagesTable({ languagesKeyByCode }).make() - logger.info('creating countires table...') - await new CountriesTable({ - countriesKeyByCode, - subdivisionsKeyByCode, - subdivisions, - countries, - cities - }).make() - logger.info('creating region table...') - await new RegionsTable({ regions }).make() - - logger.info('updating playlists.md...') - const playlists = new Markdown({ - build: `${ROOT_DIR}/PLAYLISTS.md`, - template: `${README_DIR}/template.md` - }) - playlists.compile() -} - -main() +import { CategoriesTable, CountriesTable, LanguagesTable, RegionsTable } from '../../tables' +import { README_DIR, ROOT_DIR } from '../../constants' +import { Logger } from '@freearhey/core' +import { Markdown } from '../../core' +import { loadData } from '../../api' + +async function main() { + const logger = new Logger() + + logger.info('loading data from api...') + await loadData() + + logger.info('creating category table...') + await new CategoriesTable().create() + logger.info('creating language table...') + await new LanguagesTable().create() + logger.info('creating countires table...') + await new CountriesTable().create() + logger.info('creating region table...') + await new RegionsTable().create() + + logger.info('updating playlists.md...') + const playlists = new Markdown({ + build: `${ROOT_DIR}/PLAYLISTS.md`, + template: `${README_DIR}/template.md` + }) + playlists.compile() +} + +main() diff --git a/scripts/commands/report/create.ts b/scripts/commands/report/create.ts index 176681bd6e..e7c31704d9 100644 --- a/scripts/commands/report/create.ts +++ b/scripts/commands/report/create.ts @@ -1,178 +1,159 @@ -import { DataLoader, DataProcessor, IssueLoader, PlaylistParser } from '../../core' -import { Logger, Storage, Collection, Dictionary } from '@freearhey/core' -import { DataProcessorData } from '../../types/dataProcessor' -import { DATA_DIR, STREAMS_DIR } from '../../constants' -import { DataLoaderData } from '../../types/dataLoader' -import { Issue, Stream } from '../../models' -import { isURI } from '../../utils' - -async function main() { - const logger = new Logger() - const issueLoader = new IssueLoader() - let report = new Collection() - - logger.info('loading issues...') - const issues = await issueLoader.load() - - logger.info('loading data from api...') - const processor = new DataProcessor() - const dataStorage = new Storage(DATA_DIR) - const dataLoader = new DataLoader({ storage: dataStorage }) - const data: DataLoaderData = await dataLoader.load() - const { - channelsKeyById, - feedsGroupedByChannelId, - logosGroupedByStreamId, - blocklistRecordsGroupedByChannelId - }: DataProcessorData = processor.process(data) - - logger.info('loading streams...') - const streamsStorage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ - storage: streamsStorage, - channelsKeyById, - feedsGroupedByChannelId, - logosGroupedByStreamId - }) - const files = await streamsStorage.list('**/*.m3u') - const streams = await parser.parse(files) - const streamsGroupedByUrl = streams.groupBy((stream: Stream) => stream.url) - const streamsGroupedByChannelId = streams.groupBy((stream: Stream) => stream.channelId) - const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId()) - - logger.info('checking streams:remove requests...') - const removeRequests = issues.filter(issue => - issue.labels.find((label: string) => label === 'streams:remove') - ) - removeRequests.forEach((issue: Issue) => { - const streamUrls = issue.data.getArray('streamUrl') || [] - - if (!streamUrls.length) { - const result = { - issueNumber: issue.number, - type: 'streams:remove', - streamId: undefined, - streamUrl: undefined, - status: 'missing_link' - } - - report.add(result) - } else { - for (const streamUrl of streamUrls) { - const result = { - issueNumber: issue.number, - type: 'streams:remove', - streamId: undefined, - streamUrl: truncate(streamUrl), - status: 'pending' - } - - if (streamsGroupedByUrl.missing(streamUrl)) { - result.status = 'wrong_link' - } - - report.add(result) - } - } - }) - - logger.info('checking streams:add requests...') - const addRequests = issues.filter(issue => issue.labels.includes('streams:add')) - const addRequestsBuffer = new Dictionary() - addRequests.forEach((issue: Issue) => { - const streamId = issue.data.getString('streamId') || '' - const streamUrl = issue.data.getString('streamUrl') || '' - const [channelId] = streamId.split('@') - - const result = { - issueNumber: issue.number, - type: 'streams:add', - streamId: streamId || undefined, - streamUrl: truncate(streamUrl), - status: 'pending' - } - - if (!channelId) result.status = 'missing_id' - else if (!streamUrl) result.status = 'missing_link' - else if (!isURI(streamUrl)) result.status = 'invalid_link' - else if (blocklistRecordsGroupedByChannelId.has(channelId)) result.status = 'blocked' - else if (channelsKeyById.missing(channelId)) result.status = 'wrong_id' - else if (streamsGroupedByUrl.has(streamUrl)) result.status = 'on_playlist' - else if (addRequestsBuffer.has(streamUrl)) result.status = 'duplicate' - else result.status = 'pending' - - addRequestsBuffer.set(streamUrl, true) - - report.add(result) - }) - - logger.info('checking streams:edit requests...') - const editRequests = issues.filter(issue => - issue.labels.find((label: string) => label === 'streams:edit') - ) - editRequests.forEach((issue: Issue) => { - const streamId = issue.data.getString('streamId') || '' - const streamUrl = issue.data.getString('streamUrl') || '' - const [channelId] = streamId.split('@') - - const result = { - issueNumber: issue.number, - type: 'streams:edit', - streamId: streamId || undefined, - streamUrl: truncate(streamUrl), - status: 'pending' - } - - if (!streamUrl) result.status = 'missing_link' - else if (streamsGroupedByUrl.missing(streamUrl)) result.status = 'invalid_link' - else if (channelId && channelsKeyById.missing(channelId)) result.status = 'invalid_id' - - report.add(result) - }) - - logger.info('checking channel search requests...') - const channelSearchRequests = issues.filter(issue => - issue.labels.find((label: string) => label === 'channel search') - ) - const channelSearchRequestsBuffer = new Dictionary() - channelSearchRequests.forEach((issue: Issue) => { - const streamId = issue.data.getString('channelId') || '' - const [channelId, feedId] = streamId.split('@') - - const result = { - issueNumber: issue.number, - type: 'channel search', - streamId: streamId || undefined, - streamUrl: undefined, - status: 'pending' - } - - if (!channelId) result.status = 'missing_id' - else if (channelsKeyById.missing(channelId)) result.status = 'invalid_id' - else if (channelSearchRequestsBuffer.has(streamId)) result.status = 'duplicate' - else if (blocklistRecordsGroupedByChannelId.has(channelId)) result.status = 'blocked' - else if (streamsGroupedById.has(streamId)) result.status = 'fulfilled' - else if (!feedId && streamsGroupedByChannelId.has(channelId)) result.status = 'fulfilled' - else { - const channelData = channelsKeyById.get(channelId) - if (channelData && channelData.isClosed) result.status = 'closed' - } - - channelSearchRequestsBuffer.set(streamId, true) - - report.add(result) - }) - - report = report.orderBy(item => item.issueNumber).filter(item => item.status !== 'pending') - - console.table(report.all()) -} - -main() - -function truncate(string: string, limit: number = 100) { - if (!string) return string - if (string.length < limit) return string - - return string.slice(0, limit) + '...' -} +import { Logger, Collection, Dictionary } from '@freearhey/core' +import { IssueLoader, PlaylistParser } from '../../core' +import { Storage } from '@freearhey/storage-js' +import { isURI, truncate } from '../../utils' +import { STREAMS_DIR } from '../../constants' +import { Issue, Stream } from '../../models' +import { data, loadData } from '../../api' + +async function main() { + const logger = new Logger() + const issueLoader = new IssueLoader() + let report = new Collection() + + logger.info('loading issues...') + const issues = await issueLoader.load() + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ + storage: streamsStorage + }) + const files = await streamsStorage.list('**/*.m3u') + const streams = await parser.parse(files) + const streamsGroupedByUrl = streams.groupBy((stream: Stream) => stream.url) + const streamsGroupedByChannel = streams.groupBy((stream: Stream) => stream.channel) + const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId()) + + logger.info('checking streams:remove requests...') + const removeRequests = issues.filter(issue => + issue.labels.find((label: string) => label === 'streams:remove') + ) + removeRequests.forEach((issue: Issue) => { + const streamUrls = issue.data.getArray('streamUrl') || [] + + if (!streamUrls.length) { + const result = { + issueNumber: issue.number, + type: 'streams:remove', + streamId: undefined, + streamUrl: undefined, + status: 'missing_link' + } + + report.add(result) + } else { + for (const streamUrl of streamUrls) { + const result = { + issueNumber: issue.number, + type: 'streams:remove', + streamId: undefined, + streamUrl: truncate(streamUrl), + status: 'pending' + } + + if (streamsGroupedByUrl.missing(streamUrl)) { + result.status = 'wrong_link' + } + + report.add(result) + } + } + }) + + logger.info('checking streams:add requests...') + const addRequests = issues.filter(issue => issue.labels.includes('streams:add')) + const addRequestsBuffer = new Dictionary() + addRequests.forEach((issue: Issue) => { + const streamId = issue.data.getString('streamId') || '' + const streamUrl = issue.data.getString('streamUrl') || '' + const [channelId] = streamId.split('@') + + const result = { + issueNumber: issue.number, + type: 'streams:add', + streamId: streamId || undefined, + streamUrl: truncate(streamUrl), + status: 'pending' + } + + if (!channelId) result.status = 'missing_id' + else if (!streamUrl) result.status = 'missing_link' + else if (!isURI(streamUrl)) result.status = 'invalid_link' + else if (data.blocklistRecordsGroupedByChannel.has(channelId)) result.status = 'blocked' + else if (data.channelsKeyById.missing(channelId)) result.status = 'wrong_id' + else if (streamsGroupedByUrl.has(streamUrl)) result.status = 'on_playlist' + else if (addRequestsBuffer.has(streamUrl)) result.status = 'duplicate' + else result.status = 'pending' + + addRequestsBuffer.set(streamUrl, true) + + report.add(result) + }) + + logger.info('checking streams:edit requests...') + const editRequests = issues.filter(issue => + issue.labels.find((label: string) => label === 'streams:edit') + ) + editRequests.forEach((issue: Issue) => { + const streamId = issue.data.getString('streamId') || '' + const streamUrl = issue.data.getString('streamUrl') || '' + const [channelId] = streamId.split('@') + + const result = { + issueNumber: issue.number, + type: 'streams:edit', + streamId: streamId || undefined, + streamUrl: truncate(streamUrl), + status: 'pending' + } + + if (!streamUrl) result.status = 'missing_link' + else if (streamsGroupedByUrl.missing(streamUrl)) result.status = 'invalid_link' + else if (channelId && data.channelsKeyById.missing(channelId)) result.status = 'invalid_id' + + report.add(result) + }) + + logger.info('checking channel search requests...') + const channelSearchRequests = issues.filter(issue => + issue.labels.find((label: string) => label === 'channel search') + ) + const channelSearchRequestsBuffer = new Dictionary() + channelSearchRequests.forEach((issue: Issue) => { + const streamId = issue.data.getString('channelId') || '' + const [channelId, feedId] = streamId.split('@') + + const result = { + issueNumber: issue.number, + type: 'channel search', + streamId: streamId || undefined, + streamUrl: undefined, + status: 'pending' + } + + if (!channelId) result.status = 'missing_id' + else if (data.channelsKeyById.missing(channelId)) result.status = 'invalid_id' + else if (channelSearchRequestsBuffer.has(streamId)) result.status = 'duplicate' + else if (data.blocklistRecordsGroupedByChannel.has(channelId)) result.status = 'blocked' + else if (streamsGroupedById.has(streamId)) result.status = 'fulfilled' + else if (!feedId && streamsGroupedByChannel.has(channelId)) result.status = 'fulfilled' + else { + const channelData = data.channelsKeyById.get(channelId) + if (channelData && channelData.isClosed()) result.status = 'closed' + } + + channelSearchRequestsBuffer.set(streamId, true) + + report.add(result) + }) + + report = report.sortBy(item => item.issueNumber).filter(item => item.status !== 'pending') + + console.table(report.all()) +} + +main() diff --git a/scripts/core/apiClient.ts b/scripts/core/apiClient.ts deleted file mode 100644 index e4815a81aa..0000000000 --- a/scripts/core/apiClient.ts +++ /dev/null @@ -1,16 +0,0 @@ -import axios, { AxiosInstance, AxiosResponse, AxiosRequestConfig } from 'axios' - -export class ApiClient { - instance: AxiosInstance - - constructor() { - this.instance = axios.create({ - baseURL: 'https://iptv-org.github.io/api', - responseType: 'stream' - }) - } - - get(url: string, options: AxiosRequestConfig): Promise { - return this.instance.get(url, options) - } -} diff --git a/scripts/core/cliTable.ts b/scripts/core/cliTable.ts index 61d9e608e6..bf8e20b81e 100644 --- a/scripts/core/cliTable.ts +++ b/scripts/core/cliTable.ts @@ -1,22 +1,22 @@ -import { Table } from 'console-table-printer' -import { ComplexOptions } from 'console-table-printer/dist/src/models/external-table' - -export class CliTable { - table: Table - - constructor(options?: ComplexOptions | string[]) { - this.table = new Table(options) - } - - append(row) { - this.table.addRow(row) - } - - render() { - this.table.printTable() - } - - toString() { - return this.table.render() - } -} +import { ComplexOptions } from 'console-table-printer/dist/src/models/external-table' +import { Table } from 'console-table-printer' + +export class CliTable { + table: Table + + constructor(options?: ComplexOptions | string[]) { + this.table = new Table(options) + } + + append(row) { + this.table.addRow(row) + } + + render() { + this.table.printTable() + } + + toString() { + return this.table.render() + } +} diff --git a/scripts/core/dataLoader.ts b/scripts/core/dataLoader.ts deleted file mode 100644 index 89b45c00c0..0000000000 --- a/scripts/core/dataLoader.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { ApiClient } from './apiClient' -import { Storage } from '@freearhey/core' -import cliProgress, { MultiBar } from 'cli-progress' -import type { DataLoaderProps, DataLoaderData } from '../types/dataLoader' - -const formatBytes = (bytes: number) => { - if (bytes === 0) return '0 B' - const k = 1024 - const sizes = ['B', 'KB', 'MB', 'GB'] - const i = Math.floor(Math.log(bytes) / Math.log(k)) - return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i] -} - -export class DataLoader { - client: ApiClient - storage: Storage - progressBar: MultiBar - - constructor(props: DataLoaderProps) { - this.client = new ApiClient() - this.storage = props.storage - this.progressBar = new cliProgress.MultiBar({ - stopOnComplete: true, - hideCursor: true, - forceRedraw: true, - barsize: 36, - format(options, params, payload) { - const filename = payload.filename.padEnd(18, ' ') - const barsize = options.barsize || 40 - const percent = (params.progress * 100).toFixed(2) - const speed = payload.speed ? formatBytes(payload.speed) + '/s' : 'N/A' - const total = formatBytes(params.total) - const completeSize = Math.round(params.progress * barsize) - const incompleteSize = barsize - completeSize - const bar = - options.barCompleteString && options.barIncompleteString - ? options.barCompleteString.substr(0, completeSize) + - options.barGlue + - options.barIncompleteString.substr(0, incompleteSize) - : '-'.repeat(barsize) - - return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}` - } - }) - } - - async load(): Promise { - const [ - countries, - regions, - subdivisions, - languages, - categories, - blocklist, - channels, - feeds, - logos, - timezones, - guides, - streams, - cities - ] = await Promise.all([ - this.storage.json('countries.json'), - this.storage.json('regions.json'), - this.storage.json('subdivisions.json'), - this.storage.json('languages.json'), - this.storage.json('categories.json'), - this.storage.json('blocklist.json'), - this.storage.json('channels.json'), - this.storage.json('feeds.json'), - this.storage.json('logos.json'), - this.storage.json('timezones.json'), - this.storage.json('guides.json'), - this.storage.json('streams.json'), - this.storage.json('cities.json') - ]) - - return { - countries, - regions, - subdivisions, - languages, - categories, - blocklist, - channels, - feeds, - logos, - timezones, - guides, - streams, - cities - } - } - - async download(filename: string) { - if (!this.storage || !this.progressBar) return - - const stream = await this.storage.createStream(filename) - const progressBar = this.progressBar.create(0, 0, { filename }) - - this.client - .get(filename, { - responseType: 'stream', - onDownloadProgress({ total, loaded, rate }) { - if (total) progressBar.setTotal(total) - progressBar.update(loaded, { speed: rate }) - } - }) - .then(response => { - response.data.pipe(stream) - }) - } -} diff --git a/scripts/core/dataProcessor.ts b/scripts/core/dataProcessor.ts deleted file mode 100644 index e7fd94dd5d..0000000000 --- a/scripts/core/dataProcessor.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { DataProcessorData } from '../types/dataProcessor' -import { DataLoaderData } from '../types/dataLoader' -import { Collection } from '@freearhey/core' -import { - BlocklistRecord, - Subdivision, - Category, - Language, - Timezone, - Channel, - Country, - Region, - Stream, - Guide, - City, - Feed, - Logo -} from '../models' - -export class DataProcessor { - process(data: DataLoaderData): DataProcessorData { - let regions = new Collection(data.regions).map(data => new Region(data)) - let regionsKeyByCode = regions.keyBy((region: Region) => region.code) - - const categories = new Collection(data.categories).map(data => new Category(data)) - const categoriesKeyById = categories.keyBy((category: Category) => category.id) - - const languages = new Collection(data.languages).map(data => new Language(data)) - const languagesKeyByCode = languages.keyBy((language: Language) => language.code) - - let subdivisions = new Collection(data.subdivisions).map(data => new Subdivision(data)) - let subdivisionsKeyByCode = subdivisions.keyBy((subdivision: Subdivision) => subdivision.code) - let subdivisionsGroupedByCountryCode = subdivisions.groupBy( - (subdivision: Subdivision) => subdivision.countryCode - ) - - let countries = new Collection(data.countries).map(data => new Country(data)) - let countriesKeyByCode = countries.keyBy((country: Country) => country.code) - - const cities = new Collection(data.cities).map(data => - new City(data) - .withRegions(regions) - .withCountry(countriesKeyByCode) - .withSubdivision(subdivisionsKeyByCode) - ) - const citiesKeyByCode = cities.keyBy((city: City) => city.code) - const citiesGroupedByCountryCode = cities.groupBy((city: City) => city.countryCode) - const citiesGroupedBySubdivisionCode = cities.groupBy((city: City) => city.subdivisionCode) - - const timezones = new Collection(data.timezones).map(data => - new Timezone(data).withCountries(countriesKeyByCode) - ) - const timezonesKeyById = timezones.keyBy((timezone: Timezone) => timezone.id) - - const blocklistRecords = new Collection(data.blocklist).map(data => new BlocklistRecord(data)) - const blocklistRecordsGroupedByChannelId = blocklistRecords.groupBy( - (blocklistRecord: BlocklistRecord) => blocklistRecord.channelId - ) - - let channels = new Collection(data.channels).map(data => new Channel(data)) - let channelsKeyById = channels.keyBy((channel: Channel) => channel.id) - - let feeds = new Collection(data.feeds).map(data => new Feed(data)) - let feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId) - let feedsGroupedById = feeds.groupBy((feed: Feed) => feed.id) - - const logos = new Collection(data.logos).map(data => new Logo(data).withFeed(feedsGroupedById)) - const logosGroupedByChannelId = logos.groupBy((logo: Logo) => logo.channelId) - const logosGroupedByStreamId = logos.groupBy((logo: Logo) => logo.getStreamId()) - - const streams = new Collection(data.streams).map(data => - new Stream(data).withLogos(logosGroupedByStreamId) - ) - const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId()) - - const guides = new Collection(data.guides).map(data => new Guide(data)) - const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId()) - - regions = regions.map((region: Region) => - region - .withCountries(countriesKeyByCode) - .withRegions(regions) - .withSubdivisions(subdivisions) - .withCities(cities) - ) - regionsKeyByCode = regions.keyBy((region: Region) => region.code) - - countries = countries.map((country: Country) => - country - .withCities(citiesGroupedByCountryCode) - .withSubdivisions(subdivisionsGroupedByCountryCode) - .withRegions(regions) - .withLanguage(languagesKeyByCode) - ) - countriesKeyByCode = countries.keyBy((country: Country) => country.code) - - subdivisions = subdivisions.map((subdivision: Subdivision) => - subdivision - .withCities(citiesGroupedBySubdivisionCode) - .withCountry(countriesKeyByCode) - .withRegions(regions) - .withParent(subdivisionsKeyByCode) - ) - subdivisionsKeyByCode = subdivisions.keyBy((subdivision: Subdivision) => subdivision.code) - subdivisionsGroupedByCountryCode = subdivisions.groupBy( - (subdivision: Subdivision) => subdivision.countryCode - ) - - channels = channels.map((channel: Channel) => - channel - .withFeeds(feedsGroupedByChannelId) - .withLogos(logosGroupedByChannelId) - .withCategories(categoriesKeyById) - .withCountry(countriesKeyByCode) - .withSubdivision(subdivisionsKeyByCode) - .withCategories(categoriesKeyById) - ) - channelsKeyById = channels.keyBy((channel: Channel) => channel.id) - - feeds = feeds.map((feed: Feed) => - feed - .withChannel(channelsKeyById) - .withLanguages(languagesKeyByCode) - .withTimezones(timezonesKeyById) - .withBroadcastArea( - citiesKeyByCode, - subdivisionsKeyByCode, - countriesKeyByCode, - regionsKeyByCode - ) - ) - feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId) - feedsGroupedById = feeds.groupBy((feed: Feed) => feed.id) - - return { - blocklistRecordsGroupedByChannelId, - subdivisionsGroupedByCountryCode, - feedsGroupedByChannelId, - guidesGroupedByStreamId, - logosGroupedByStreamId, - subdivisionsKeyByCode, - countriesKeyByCode, - languagesKeyByCode, - streamsGroupedById, - categoriesKeyById, - timezonesKeyById, - regionsKeyByCode, - blocklistRecords, - channelsKeyById, - citiesKeyByCode, - subdivisions, - categories, - countries, - languages, - timezones, - channels, - regions, - streams, - cities, - guides, - feeds, - logos - } - } -} diff --git a/scripts/core/htmlTable.ts b/scripts/core/htmlTable.ts index b4f7b42ac6..f90dd4b913 100644 --- a/scripts/core/htmlTable.ts +++ b/scripts/core/htmlTable.ts @@ -1,46 +1,50 @@ -type Column = { - name: string - nowrap?: boolean - align?: string -} - -type DataItem = string[] - -export class HTMLTable { - data: DataItem[] - columns: Column[] - - constructor(data: DataItem[], columns: Column[]) { - this.data = data - this.columns = columns - } - - toString() { - let output = '\r\n' - - output += ' \r\n ' - for (const column of this.columns) { - output += `` - } - output += '\r\n \r\n' - - output += ' \r\n' - for (const item of this.data) { - output += ' ' - let i = 0 - for (const prop in item) { - const column = this.columns[i] - const nowrap = column.nowrap ? ' nowrap' : '' - const align = column.align ? ` align="${column.align}"` : '' - output += `${item[prop]}` - i++ - } - output += '\r\n' - } - output += ' \r\n' - - output += '
${column.name}
' - - return output - } -} +import { Collection } from '@freearhey/core' + +export type HTMLTableColumn = { + name: string + nowrap?: boolean + align?: string +} + +export type HTMLTableItem = string[] + +export class HTMLTable { + data: Collection + columns: Collection + + constructor(data: Collection, columns: Collection) { + this.data = data + this.columns = columns + } + + toString() { + let output = '\r\n' + + output += ' \r\n ' + this.columns.forEach((column: HTMLTableColumn) => { + output += `` + }) + + output += '\r\n \r\n' + + output += ' \r\n' + this.data.forEach((item: HTMLTableItem) => { + output += ' ' + let i = 0 + for (const prop in item) { + const column = this.columns.all()[i] + const nowrap = column.nowrap ? ' nowrap' : '' + const align = column.align ? ` align="${column.align}"` : '' + output += `${item[prop]}` + i++ + } + output += '\r\n' + }) + + output += ' \r\n' + + output += '
${column.name}
' + + return output + } +} diff --git a/scripts/core/index.ts b/scripts/core/index.ts index 2e24771bf0..850b9d9cfb 100644 --- a/scripts/core/index.ts +++ b/scripts/core/index.ts @@ -1,14 +1,11 @@ -export * from './apiClient' -export * from './cliTable' -export * from './dataProcessor' -export * from './dataLoader' -export * from './htmlTable' -export * from './issueData' -export * from './issueLoader' -export * from './issueParser' -export * from './logParser' -export * from './markdown' -export * from './numberParser' -export * from './playlistParser' -export * from './proxyParser' -export * from './streamTester' +export * from './cliTable' +export * from './htmlTable' +export * from './issueData' +export * from './issueLoader' +export * from './issueParser' +export * from './logParser' +export * from './markdown' +export * from './numberParser' +export * from './playlistParser' +export * from './proxyParser' +export * from './streamTester' diff --git a/scripts/core/issueData.ts b/scripts/core/issueData.ts index e185e1b02f..a38209250a 100644 --- a/scripts/core/issueData.ts +++ b/scripts/core/issueData.ts @@ -1,34 +1,36 @@ -import { Dictionary } from '@freearhey/core' - -export class IssueData { - _data: Dictionary - constructor(data: Dictionary) { - this._data = data - } - - has(key: string): boolean { - return this._data.has(key) - } - - missing(key: string): boolean { - return this._data.missing(key) || this._data.get(key) === undefined - } - - getBoolean(key: string): boolean { - return Boolean(this._data.get(key)) - } - - getString(key: string): string | undefined { - const deleteSymbol = '~' - - return this._data.get(key) === deleteSymbol ? '' : this._data.get(key) - } - - getArray(key: string): string[] | undefined { - const deleteSymbol = '~' - - if (this._data.missing(key)) return undefined - - return this._data.get(key) === deleteSymbol ? [] : this._data.get(key).split('\r\n') - } -} +import { Dictionary } from '@freearhey/core' + +export class IssueData { + _data: Dictionary + constructor(data: Dictionary) { + this._data = data + } + + has(key: string): boolean { + return this._data.has(key) + } + + missing(key: string): boolean { + return this._data.missing(key) || this._data.get(key) === undefined + } + + getBoolean(key: string): boolean { + return Boolean(this._data.get(key)) + } + + getString(key: string): string | undefined { + const deleteSymbol = '~' + + return this._data.get(key) === deleteSymbol ? '' : this._data.get(key) + } + + getArray(key: string): string[] | undefined { + const deleteSymbol = '~' + + if (this._data.missing(key)) return undefined + + const value = this._data.get(key) + + return !value || value === deleteSymbol ? [] : value.split('\r\n') + } +} diff --git a/scripts/core/issueLoader.ts b/scripts/core/issueLoader.ts index e9102f25a1..43fb2185b6 100644 --- a/scripts/core/issueLoader.ts +++ b/scripts/core/issueLoader.ts @@ -1,37 +1,37 @@ -import { Collection } from '@freearhey/core' -import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods' -import { paginateRest } from '@octokit/plugin-paginate-rest' -import { Octokit } from '@octokit/core' -import { IssueParser } from './' -import { TESTING, OWNER, REPO } from '../constants' - -const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods) -const octokit = new CustomOctokit() - -export class IssueLoader { - async load(props?: { labels: string | string[] }) { - let labels = '' - if (props && props.labels) { - labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels - } - let issues: object[] = [] - if (TESTING) { - issues = (await import('../../tests/__data__/input/issues.js')).default - } else { - issues = await octokit.paginate(octokit.rest.issues.listForRepo, { - owner: OWNER, - repo: REPO, - per_page: 100, - labels, - status: 'open', - headers: { - 'X-GitHub-Api-Version': '2022-11-28' - } - }) - } - - const parser = new IssueParser() - - return new Collection(issues).map(parser.parse) - } -} +import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods' +import { paginateRest } from '@octokit/plugin-paginate-rest' +import { TESTING, OWNER, REPO } from '../constants' +import { Collection } from '@freearhey/core' +import { Octokit } from '@octokit/core' +import { IssueParser } from './' + +const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods) +const octokit = new CustomOctokit() + +export class IssueLoader { + async load(props?: { labels: string | string[] }) { + let labels = '' + if (props && props.labels) { + labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels + } + let issues: object[] = [] + if (TESTING) { + issues = (await import('../../tests/__data__/input/issues.js')).default + } else { + issues = await octokit.paginate(octokit.rest.issues.listForRepo, { + owner: OWNER, + repo: REPO, + per_page: 100, + labels, + status: 'open', + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) + } + + const parser = new IssueParser() + + return new Collection(issues).map(parser.parse) + } +} diff --git a/scripts/core/issueParser.ts b/scripts/core/issueParser.ts index b5feeaec80..ad488e6c5d 100644 --- a/scripts/core/issueParser.ts +++ b/scripts/core/issueParser.ts @@ -1,48 +1,48 @@ -import { Dictionary } from '@freearhey/core' -import { Issue } from '../models' -import { IssueData } from './issueData' - -const FIELDS = new Dictionary({ - 'Stream ID': 'streamId', - 'Channel ID': 'channelId', - 'Feed ID': 'feedId', - 'Stream URL': 'streamUrl', - 'New Stream URL': 'newStreamUrl', - Label: 'label', - Quality: 'quality', - 'HTTP User-Agent': 'httpUserAgent', - 'HTTP User Agent': 'httpUserAgent', - 'HTTP Referrer': 'httpReferrer', - 'What happened to the stream?': 'reason', - Reason: 'reason', - Notes: 'notes', - Directives: 'directives' -}) - -export class IssueParser { - parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue { - const fields = typeof issue.body === 'string' ? issue.body.split('###') : [] - - const data = new Dictionary() - fields.forEach((field: string) => { - const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : [] - let _label = parsed.shift() - _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : '' - let _value = parsed.join('\r\n') - _value = _value ? _value.trim() : '' - - if (!_label || !_value) return data - - const id: string = FIELDS.get(_label) - const value: string = _value === '_No response_' || _value === 'None' ? '' : _value - - if (!id) return - - data.set(id, value) - }) - - const labels = issue.labels.map(label => label.name) - - return new Issue({ number: issue.number, labels, data: new IssueData(data) }) - } -} +import { Dictionary } from '@freearhey/core' +import { IssueData } from './issueData' +import { Issue } from '../models' + +const FIELDS = new Dictionary({ + 'Stream ID': 'streamId', + 'Channel ID': 'channelId', + 'Feed ID': 'feedId', + 'Stream URL': 'streamUrl', + 'New Stream URL': 'newStreamUrl', + Label: 'label', + Quality: 'quality', + 'HTTP User-Agent': 'httpUserAgent', + 'HTTP User Agent': 'httpUserAgent', + 'HTTP Referrer': 'httpReferrer', + 'What happened to the stream?': 'reason', + Reason: 'reason', + Notes: 'notes', + Directives: 'directives' +}) + +export class IssueParser { + parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue { + const fields = typeof issue.body === 'string' ? issue.body.split('###') : [] + + const data = new Dictionary() + fields.forEach((field: string) => { + const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : [] + let _label = parsed.shift() + _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : '' + let _value = parsed.join('\r\n') + _value = _value ? _value.trim() : '' + + if (!_label || !_value) return data + + const id = FIELDS.get(_label) + const value: string = _value === '_No response_' || _value === 'None' ? '' : _value + + if (!id) return + + data.set(id, value) + }) + + const labels = issue.labels.map(label => label.name) + + return new Issue({ number: issue.number, labels, data: new IssueData(data) }) + } +} diff --git a/scripts/core/markdown.ts b/scripts/core/markdown.ts index e229999409..25e15095ec 100644 --- a/scripts/core/markdown.ts +++ b/scripts/core/markdown.ts @@ -1,45 +1,45 @@ -import fs from 'fs' -import path from 'path' - -type MarkdownConfig = { - build: string - template: string -} - -export class Markdown { - build: string - template: string - - constructor(config: MarkdownConfig) { - this.build = config.build - this.template = config.template - } - - compile() { - const workingDir = process.cwd() - - const templatePath = path.resolve(workingDir, this.template) - const template = fs.readFileSync(templatePath, 'utf8') - const processedContent = this.processIncludes(template, workingDir) - - if (this.build) { - const outputPath = path.resolve(workingDir, this.build) - fs.writeFileSync(outputPath, processedContent, 'utf8') - } - } - - private processIncludes(template: string, baseDir: string): string { - const includeRegex = /#include\s+"([^"]+)"/g - - return template.replace(includeRegex, (match, includePath) => { - try { - const fullPath = path.resolve(baseDir, includePath) - const includeContent = fs.readFileSync(fullPath, 'utf8') - return this.processIncludes(includeContent, baseDir) - } catch (error) { - console.warn(`Warning: Could not include file ${includePath}: ${error}`) - return match - } - }) - } -} +import path from 'path' +import fs from 'fs' + +type MarkdownConfig = { + build: string + template: string +} + +export class Markdown { + build: string + template: string + + constructor(config: MarkdownConfig) { + this.build = config.build + this.template = config.template + } + + compile() { + const workingDir = process.cwd() + + const templatePath = path.resolve(workingDir, this.template) + const template = fs.readFileSync(templatePath, 'utf8') + const processedContent = this.processIncludes(template, workingDir) + + if (this.build) { + const outputPath = path.resolve(workingDir, this.build) + fs.writeFileSync(outputPath, processedContent, 'utf8') + } + } + + private processIncludes(template: string, baseDir: string): string { + const includeRegex = /#include\s+"([^"]+)"/g + + return template.replace(includeRegex, (match, includePath) => { + try { + const fullPath = path.resolve(baseDir, includePath) + const includeContent = fs.readFileSync(fullPath, 'utf8') + return this.processIncludes(includeContent, baseDir) + } catch (error) { + console.warn(`Warning: Could not include file ${includePath}: ${error}`) + return match + } + }) + } +} diff --git a/scripts/core/playlistParser.ts b/scripts/core/playlistParser.ts index 2086bb568e..08b2542ff9 100644 --- a/scripts/core/playlistParser.ts +++ b/scripts/core/playlistParser.ts @@ -1,60 +1,43 @@ -import { Collection, Storage, Dictionary } from '@freearhey/core' -import parser from 'iptv-playlist-parser' -import { Stream } from '../models' - -type PlaylistPareserProps = { - storage: Storage - feedsGroupedByChannelId: Dictionary - logosGroupedByStreamId: Dictionary - channelsKeyById: Dictionary -} - -export class PlaylistParser { - storage: Storage - feedsGroupedByChannelId: Dictionary - logosGroupedByStreamId: Dictionary - channelsKeyById: Dictionary - - constructor({ - storage, - feedsGroupedByChannelId, - logosGroupedByStreamId, - channelsKeyById - }: PlaylistPareserProps) { - this.storage = storage - this.feedsGroupedByChannelId = feedsGroupedByChannelId - this.logosGroupedByStreamId = logosGroupedByStreamId - this.channelsKeyById = channelsKeyById - } - - async parse(files: string[]): Promise { - let streams = new Collection() - - for (const filepath of files) { - if (!this.storage.existsSync(filepath)) continue - - const _streams: Collection = await this.parseFile(filepath) - streams = streams.concat(_streams) - } - - return streams - } - - async parseFile(filepath: string): Promise { - const content = await this.storage.load(filepath) - const parsed: parser.Playlist = parser.parse(content) - - const streams = new Collection(parsed.items).map((data: parser.PlaylistItem) => { - const stream = new Stream() - .fromPlaylistItem(data) - .withFeed(this.feedsGroupedByChannelId) - .withChannel(this.channelsKeyById) - .withLogos(this.logosGroupedByStreamId) - .setFilepath(filepath) - - return stream - }) - - return streams - } -} +import { Storage } from '@freearhey/storage-js' +import { Collection } from '@freearhey/core' +import parser from 'iptv-playlist-parser' +import { Stream } from '../models' + +type PlaylistPareserProps = { + storage: Storage +} + +export class PlaylistParser { + storage: Storage + + constructor({ storage }: PlaylistPareserProps) { + this.storage = storage + } + + async parse(files: string[]): Promise> { + const parsed = new Collection() + + for (const filepath of files) { + if (!this.storage.existsSync(filepath)) continue + const _parsed: Collection = await this.parseFile(filepath) + parsed.concat(_parsed) + } + + return parsed + } + + async parseFile(filepath: string): Promise> { + const content = await this.storage.load(filepath) + const parsed: parser.Playlist = parser.parse(content) + + const streams = new Collection() + parsed.items.forEach((data: parser.PlaylistItem) => { + const stream = Stream.fromPlaylistItem(data) + stream.filepath = filepath + + streams.add(stream) + }) + + return streams + } +} diff --git a/scripts/core/streamTester.ts b/scripts/core/streamTester.ts index d5a4f5f591..acb38edc9c 100644 --- a/scripts/core/streamTester.ts +++ b/scripts/core/streamTester.ts @@ -1,117 +1,125 @@ -import { Stream } from '../models' -import { TESTING } from '../constants' -import mediaInfoFactory from 'mediainfo.js' -import axios, { AxiosInstance, AxiosProxyConfig, AxiosRequestConfig } from 'axios' -import { ProxyParser } from './proxyParser.js' -import { OptionValues } from 'commander' -import { SocksProxyAgent } from 'socks-proxy-agent' - -export type TestResult = { - status: { - ok: boolean - code: string - } -} - -export type StreamTesterProps = { - options: OptionValues -} - -export class StreamTester { - client: AxiosInstance - options: OptionValues - - constructor({ options }: StreamTesterProps) { - const proxyParser = new ProxyParser() - let request: AxiosRequestConfig = { - responseType: 'arraybuffer' - } - - if (options.proxy !== undefined) { - const proxy = proxyParser.parse(options.proxy) as AxiosProxyConfig - - if ( - proxy.protocol && - ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol)) - ) { - const socksProxyAgent = new SocksProxyAgent(options.proxy) - - request = { ...request, ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } } - } else { - request = { ...request, ...{ proxy } } - } - } - - this.client = axios.create(request) - this.options = options - } - - async test(stream: Stream): Promise { - if (TESTING) { - const results = (await import('../../tests/__data__/input/playlist_test/results.js')).default - - return results[stream.url as keyof typeof results] - } else { - try { - const res = await this.client(stream.url, { - signal: AbortSignal.timeout(this.options.timeout), - headers: { - 'User-Agent': stream.getUserAgent() || 'Mozilla/5.0', - Referer: stream.getReferrer() - } - }) - - const mediainfo = await mediaInfoFactory({ format: 'object' }) - const buffer = await res.data - const result = await mediainfo.analyzeData( - () => buffer.byteLength, - (size: any, offset: number | undefined) => - Buffer.from(buffer).subarray(offset, offset + size) - ) - - if (result && result.media && result.media.track.length > 0) { - return { - status: { - ok: true, - code: 'OK' - } - } - } else { - return { - status: { - ok: false, - code: 'NO_VIDEO' - } - } - } - } catch (error: any) { - let code = 'UNKNOWN_ERROR' - if (error.name === 'CanceledError') { - code = 'TIMEOUT' - } else if (error.name === 'AxiosError') { - if (error.response) { - const status = error.response?.status - const statusText = error.response?.statusText.toUpperCase().replace(/\s+/, '_') - code = `HTTP_${status}_${statusText}` - } else { - code = `AXIOS_${error.code}` - } - } else if (error.cause) { - const cause = error.cause as Error & { code?: string } - if (cause.code) { - code = cause.code - } else { - code = cause.name - } - } - - return { - status: { - ok: false, - code - } - } - } - } - } -} +import axios, { AxiosInstance, AxiosProxyConfig, AxiosRequestConfig, AxiosResponse } from 'axios' +import { SocksProxyAgent } from 'socks-proxy-agent' +import { ProxyParser } from './proxyParser.js' +import mediaInfoFactory from 'mediainfo.js' +import { OptionValues } from 'commander' +import { TESTING } from '../constants' +import { Stream } from '../models' + +export type StreamTesterResult = { + status: { + ok: boolean + code: string + } +} + +export type StreamTesterError = { + name: string + code?: string + cause?: Error & { code?: string } + response?: AxiosResponse +} + +export type StreamTesterProps = { + options: OptionValues +} + +export class StreamTester { + client: AxiosInstance + options: OptionValues + + constructor({ options }: StreamTesterProps) { + const proxyParser = new ProxyParser() + let request: AxiosRequestConfig = { + responseType: 'arraybuffer' + } + + if (options.proxy !== undefined) { + const proxy = proxyParser.parse(options.proxy) as AxiosProxyConfig + + if ( + proxy.protocol && + ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol)) + ) { + const socksProxyAgent = new SocksProxyAgent(options.proxy) + + request = { ...request, ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } } + } else { + request = { ...request, ...{ proxy } } + } + } + + this.client = axios.create(request) + this.options = options + } + + async test(stream: Stream): Promise { + if (TESTING) { + const results = (await import('../../tests/__data__/input/playlist_test/results.js')).default + + return results[stream.url as keyof typeof results] + } else { + try { + const res = await this.client(stream.url, { + signal: AbortSignal.timeout(this.options.timeout), + headers: { + 'User-Agent': stream.user_agent || 'Mozilla/5.0', + Referer: stream.referrer + } + }) + + const mediainfo = await mediaInfoFactory({ format: 'object' }) + const buffer = await res.data + const result = await mediainfo.analyzeData( + () => buffer.byteLength, + (size: number, offset: number) => Buffer.from(buffer).subarray(offset, offset + size) + ) + + if (result && result.media && result.media.track.length > 0) { + return { + status: { + ok: true, + code: 'OK' + } + } + } else { + return { + status: { + ok: false, + code: 'NO_VIDEO' + } + } + } + } catch (err: unknown) { + const error = err as StreamTesterError + + let code = 'UNKNOWN_ERROR' + if (error.name === 'CanceledError') { + code = 'TIMEOUT' + } else if (error.name === 'AxiosError') { + if (error.response) { + const status = error.response?.status + const statusText = error.response?.statusText.toUpperCase().replace(/\s+/, '_') + code = `HTTP_${status}_${statusText}` + } else { + code = `AXIOS_${error.code}` + } + } else if (error.cause) { + const cause = error.cause + if (cause.code) { + code = cause.code + } else { + code = cause.name + } + } + + return { + status: { + ok: false, + code + } + } + } + } + } +} diff --git a/scripts/generators/categoriesGenerator.ts b/scripts/generators/categoriesGenerator.ts index 83eb4b5255..d42719392e 100644 --- a/scripts/generators/categoriesGenerator.ts +++ b/scripts/generators/categoriesGenerator.ts @@ -1,54 +1,60 @@ -import { Collection, Storage, File } from '@freearhey/core' -import { Stream, Category, Playlist } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type CategoriesGeneratorProps = { - streams: Collection - categories: Collection - logFile: File -} - -export class CategoriesGenerator implements Generator { - streams: Collection - categories: Collection - storage: Storage - logFile: File - - constructor({ streams, categories, logFile }: CategoriesGeneratorProps) { - this.streams = streams.clone() - this.categories = categories - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate() { - const streams = this.streams.orderBy([(stream: Stream) => stream.getTitle()]) - - this.categories.forEach(async (category: Category) => { - const categoryStreams = streams - .filter((stream: Stream) => stream.hasCategory(category)) - .map((stream: Stream) => { - const groupTitle = stream.getCategoryNames().join(';') - if (groupTitle) stream.groupTitle = groupTitle - - return stream - }) - - const playlist = new Playlist(categoryStreams, { public: true }) - const filepath = `categories/${category.id}.m3u` - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + EOL - ) - }) - - const undefinedStreams = streams.filter((stream: Stream) => !stream.hasCategories()) - const playlist = new Playlist(undefinedStreams, { public: true }) - const filepath = 'categories/undefined.m3u' - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + EOL - ) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Collection } from '@freearhey/core' +import { Stream, Playlist } from '../models' +import { Generator } from './generator' +import * as sdk from '@iptv-org/sdk' + +type CategoriesGeneratorProps = { + streams: Collection + categories: Collection + logFile: File +} + +export class CategoriesGenerator implements Generator { + streams: Collection + categories: Collection + storage: Storage + logFile: File + + constructor({ streams, categories, logFile }: CategoriesGeneratorProps) { + this.streams = streams.clone() + this.categories = categories + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate() { + const streams = this.streams.sortBy([(stream: Stream) => stream.title]) + + this.categories.forEach(async (category: sdk.Models.Category) => { + const categoryStreams = streams + .filter((stream: Stream) => stream.hasCategory(category)) + .map((stream: Stream) => { + const groupTitle = stream + .getCategories() + .map(category => category.name) + .sort() + .join(';') + if (groupTitle) stream.groupTitle = groupTitle + + return stream + }) + + const playlist = new Playlist(categoryStreams, { public: true }) + const filepath = `categories/${category.id}.m3u` + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + EOL + ) + }) + + const undefinedStreams = streams.filter((stream: Stream) => stream.getCategories().isEmpty()) + const playlist = new Playlist(undefinedStreams, { public: true }) + const filepath = 'categories/undefined.m3u' + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + EOL + ) + } +} diff --git a/scripts/generators/citiesGenerator.ts b/scripts/generators/citiesGenerator.ts index 7d0891fbc8..237e8da3dc 100644 --- a/scripts/generators/citiesGenerator.ts +++ b/scripts/generators/citiesGenerator.ts @@ -1,43 +1,54 @@ -import { City, Stream, Playlist } from '../models' -import { Collection, Storage, File } from '@freearhey/core' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type CitiesGeneratorProps = { - streams: Collection - cities: Collection - logFile: File -} - -export class CitiesGenerator implements Generator { - streams: Collection - cities: Collection - storage: Storage - logFile: File - - constructor({ streams, cities, logFile }: CitiesGeneratorProps) { - this.streams = streams.clone() - this.cities = cities - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - const streams = this.streams - .orderBy((stream: Stream) => stream.getTitle()) - .filter((stream: Stream) => stream.isSFW()) - - this.cities.forEach(async (city: City) => { - const cityStreams = streams.filter((stream: Stream) => stream.isBroadcastInCity(city)) - - if (cityStreams.isEmpty()) return - - const playlist = new Playlist(cityStreams, { public: true }) - const filepath = `cities/${city.code.toLowerCase()}.m3u` - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'city', filepath, count: playlist.streams.count() }) + EOL - ) - }) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Stream, Playlist } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' +import * as sdk from '@iptv-org/sdk' + +type CitiesGeneratorProps = { + streams: Collection + cities: Collection + logFile: File +} + +export class CitiesGenerator implements Generator { + streams: Collection + cities: Collection + storage: Storage + logFile: File + + constructor({ streams, cities, logFile }: CitiesGeneratorProps) { + this.streams = streams.clone() + this.cities = cities + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate(): Promise { + const streams = this.streams + .sortBy((stream: Stream) => stream.title) + .filter((stream: Stream) => stream.isSFW()) + + const streamsGroupedByCityCode = {} + streams.forEach((stream: Stream) => { + stream.getBroadcastCities().forEach((city: sdk.Models.City) => { + if (streamsGroupedByCityCode[city.code]) { + streamsGroupedByCityCode[city.code].add(stream) + } else { + streamsGroupedByCityCode[city.code] = new Collection([stream]) + } + }) + }) + + for (const cityCode in streamsGroupedByCityCode) { + const cityStreams = streamsGroupedByCityCode[cityCode] + + const playlist = new Playlist(cityStreams, { public: true }) + const filepath = `cities/${cityCode.toLowerCase()}.m3u` + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'city', filepath, count: playlist.streams.count() }) + EOL + ) + } + } +} diff --git a/scripts/generators/countriesGenerator.ts b/scripts/generators/countriesGenerator.ts index 39d7612582..0ee466dc62 100644 --- a/scripts/generators/countriesGenerator.ts +++ b/scripts/generators/countriesGenerator.ts @@ -1,68 +1,80 @@ -import { Country, Stream, Playlist } from '../models' -import { Collection, Storage, File } from '@freearhey/core' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type CountriesGeneratorProps = { - streams: Collection - countries: Collection - logFile: File -} - -export class CountriesGenerator implements Generator { - streams: Collection - countries: Collection - storage: Storage - logFile: File - - constructor({ streams, countries, logFile }: CountriesGeneratorProps) { - this.streams = streams.clone() - this.countries = countries - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - const streams = this.streams - .orderBy((stream: Stream) => stream.getTitle()) - .filter((stream: Stream) => stream.isSFW()) - - this.countries.forEach(async (country: Country) => { - const countryStreams = streams.filter((stream: Stream) => - stream.isBroadcastInCountry(country) - ) - if (countryStreams.isEmpty()) return - - const playlist = new Playlist(countryStreams, { public: true }) - const filepath = `countries/${country.code.toLowerCase()}.m3u` - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'country', filepath, count: playlist.streams.count() }) + EOL - ) - }) - - const internationalStreams = streams.filter((stream: Stream) => stream.isInternational()) - const internationalPlaylist = new Playlist(internationalStreams, { public: true }) - const internationalFilepath = 'countries/int.m3u' - await this.storage.save(internationalFilepath, internationalPlaylist.toString()) - this.logFile.append( - JSON.stringify({ - type: 'country', - filepath: internationalFilepath, - count: internationalPlaylist.streams.count() - }) + EOL - ) - - const undefinedStreams = streams.filter((stream: Stream) => !stream.hasBroadcastArea()) - const undefinedPlaylist = new Playlist(undefinedStreams, { public: true }) - const undefinedFilepath = 'countries/undefined.m3u' - await this.storage.save(undefinedFilepath, undefinedPlaylist.toString()) - this.logFile.append( - JSON.stringify({ - type: 'country', - filepath: undefinedFilepath, - count: undefinedPlaylist.streams.count() - }) + EOL - ) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Stream, Playlist } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' +import * as sdk from '@iptv-org/sdk' + +type CountriesGeneratorProps = { + streams: Collection + countries: Collection + logFile: File +} + +export class CountriesGenerator implements Generator { + streams: Collection + countries: Collection + storage: Storage + logFile: File + + constructor({ streams, countries, logFile }: CountriesGeneratorProps) { + this.streams = streams.clone() + this.countries = countries + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate(): Promise { + const streams = this.streams + .sortBy((stream: Stream) => stream.title) + .filter((stream: Stream) => stream.isSFW()) + + const streamsGroupedByCountryCode = {} + streams.forEach((stream: Stream) => { + stream.getBroadcastCountries().forEach((country: sdk.Models.Country) => { + if (streamsGroupedByCountryCode[country.code]) { + streamsGroupedByCountryCode[country.code].add(stream) + } else { + streamsGroupedByCountryCode[country.code] = new Collection([stream]) + } + }) + }) + + for (const countryCode in streamsGroupedByCountryCode) { + const countryStreams = streamsGroupedByCountryCode[countryCode] + + const playlist = new Playlist(countryStreams, { public: true }) + const filepath = `countries/${countryCode.toLowerCase()}.m3u` + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'country', filepath, count: playlist.streams.count() }) + EOL + ) + } + + const internationalStreams = streams.filter((stream: Stream) => stream.isInternational()) + const internationalPlaylist = new Playlist(internationalStreams, { public: true }) + const internationalFilepath = 'countries/int.m3u' + await this.storage.save(internationalFilepath, internationalPlaylist.toString()) + this.logFile.append( + JSON.stringify({ + type: 'country', + filepath: internationalFilepath, + count: internationalPlaylist.streams.count() + }) + EOL + ) + + const undefinedStreams = streams.filter((stream: Stream) => + stream.getBroadcastAreaCodes().isEmpty() + ) + const undefinedPlaylist = new Playlist(undefinedStreams, { public: true }) + const undefinedFilepath = 'countries/undefined.m3u' + await this.storage.save(undefinedFilepath, undefinedPlaylist.toString()) + this.logFile.append( + JSON.stringify({ + type: 'country', + filepath: undefinedFilepath, + count: undefinedPlaylist.streams.count() + }) + EOL + ) + } +} diff --git a/scripts/generators/index.ts b/scripts/generators/index.ts index 66cf94eb4e..1cfd8a38ec 100644 --- a/scripts/generators/index.ts +++ b/scripts/generators/index.ts @@ -1,13 +1,12 @@ -export * from './categoriesGenerator' -export * from './citiesGenerator' -export * from './countriesGenerator' -export * from './indexCategoryGenerator' -export * from './indexCountryGenerator' -export * from './indexGenerator' -export * from './indexLanguageGenerator' -export * from './indexNsfwGenerator' -export * from './languagesGenerator' -export * from './rawGenerator' -export * from './regionsGenerator' -export * from './sourcesGenerator' -export * from './subdivisionsGenerator' +export * from './categoriesGenerator' +export * from './citiesGenerator' +export * from './countriesGenerator' +export * from './indexCategoryGenerator' +export * from './indexCountryGenerator' +export * from './indexGenerator' +export * from './indexLanguageGenerator' +export * from './languagesGenerator' +export * from './rawGenerator' +export * from './regionsGenerator' +export * from './sourcesGenerator' +export * from './subdivisionsGenerator' diff --git a/scripts/generators/indexCategoryGenerator.ts b/scripts/generators/indexCategoryGenerator.ts index 754a7d22f1..08ebf74be8 100644 --- a/scripts/generators/indexCategoryGenerator.ts +++ b/scripts/generators/indexCategoryGenerator.ts @@ -1,55 +1,56 @@ -import { Collection, Storage, File } from '@freearhey/core' -import { Stream, Playlist, Category } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type IndexCategoryGeneratorProps = { - streams: Collection - logFile: File -} - -export class IndexCategoryGenerator implements Generator { - streams: Collection - storage: Storage - logFile: File - - constructor({ streams, logFile }: IndexCategoryGeneratorProps) { - this.streams = streams.clone() - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - const streams = this.streams - .orderBy(stream => stream.getTitle()) - .filter(stream => stream.isSFW()) - - let groupedStreams = new Collection() - streams.forEach((stream: Stream) => { - if (!stream.hasCategories()) { - const streamClone = stream.clone() - streamClone.groupTitle = 'Undefined' - groupedStreams.add(streamClone) - return - } - - stream.getCategories().forEach((category: Category) => { - const streamClone = stream.clone() - streamClone.groupTitle = category.name - groupedStreams.push(streamClone) - }) - }) - - groupedStreams = groupedStreams.orderBy(stream => { - if (stream.groupTitle === 'Undefined') return 'ZZ' - return stream.groupTitle - }) - - const playlist = new Playlist(groupedStreams, { public: true }) - const filepath = 'index.category.m3u' - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL - ) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Stream, Playlist } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' +import * as sdk from '@iptv-org/sdk' + +type IndexCategoryGeneratorProps = { + streams: Collection + logFile: File +} + +export class IndexCategoryGenerator implements Generator { + streams: Collection + storage: Storage + logFile: File + + constructor({ streams, logFile }: IndexCategoryGeneratorProps) { + this.streams = streams.clone() + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate(): Promise { + const streams = this.streams.sortBy(stream => stream.title).filter(stream => stream.isSFW()) + + let groupedStreams = new Collection() + streams.forEach((stream: Stream) => { + const streamCategories = stream.getCategories() + if (streamCategories.isEmpty()) { + const streamClone = stream.clone() + streamClone.groupTitle = 'Undefined' + groupedStreams.add(streamClone) + return + } + + streamCategories.forEach((category: sdk.Models.Category) => { + const streamClone = stream.clone() + streamClone.groupTitle = category.name + groupedStreams.add(streamClone) + }) + }) + + groupedStreams = groupedStreams.sortBy(stream => { + if (stream.groupTitle === 'Undefined') return 'ZZ' + return stream.groupTitle + }) + + const playlist = new Playlist(groupedStreams, { public: true }) + const filepath = 'index.category.m3u' + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL + ) + } +} diff --git a/scripts/generators/indexCountryGenerator.ts b/scripts/generators/indexCountryGenerator.ts index 016e86f731..078a1f9d6a 100644 --- a/scripts/generators/indexCountryGenerator.ts +++ b/scripts/generators/indexCountryGenerator.ts @@ -1,63 +1,67 @@ -import { Collection, Storage, File } from '@freearhey/core' -import { Stream, Playlist, Country } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type IndexCountryGeneratorProps = { - streams: Collection - logFile: File -} - -export class IndexCountryGenerator implements Generator { - streams: Collection - storage: Storage - logFile: File - - constructor({ streams, logFile }: IndexCountryGeneratorProps) { - this.streams = streams.clone() - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - let groupedStreams = new Collection() - - this.streams - .orderBy((stream: Stream) => stream.getTitle()) - .filter((stream: Stream) => stream.isSFW()) - .forEach((stream: Stream) => { - if (!stream.hasBroadcastArea()) { - const streamClone = stream.clone() - streamClone.groupTitle = 'Undefined' - groupedStreams.add(streamClone) - return - } - - stream.getBroadcastCountries().forEach((country: Country) => { - const streamClone = stream.clone() - streamClone.groupTitle = country.name - groupedStreams.add(streamClone) - }) - - if (stream.isInternational()) { - const streamClone = stream.clone() - streamClone.groupTitle = 'International' - groupedStreams.add(streamClone) - } - }) - - groupedStreams = groupedStreams.orderBy((stream: Stream) => { - if (stream.groupTitle === 'International') return 'ZZ' - if (stream.groupTitle === 'Undefined') return 'ZZZ' - - return stream.groupTitle - }) - - const playlist = new Playlist(groupedStreams, { public: true }) - const filepath = 'index.country.m3u' - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL - ) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Stream, Playlist } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' +import * as sdk from '@iptv-org/sdk' + +type IndexCountryGeneratorProps = { + streams: Collection + logFile: File +} + +export class IndexCountryGenerator implements Generator { + streams: Collection + storage: Storage + logFile: File + + constructor({ streams, logFile }: IndexCountryGeneratorProps) { + this.streams = streams.clone() + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate(): Promise { + let groupedStreams = new Collection() + + this.streams + .sortBy((stream: Stream) => stream.title) + .filter((stream: Stream) => stream.isSFW()) + .forEach((stream: Stream) => { + const broadcastAreaCountries = stream.getBroadcastCountries() + + if (stream.getBroadcastAreaCodes().isEmpty()) { + const streamClone = stream.clone() + streamClone.groupTitle = 'Undefined' + groupedStreams.add(streamClone) + return + } + + broadcastAreaCountries.forEach((country: sdk.Models.Country) => { + const streamClone = stream.clone() + streamClone.groupTitle = country.name + groupedStreams.add(streamClone) + }) + + if (stream.isInternational()) { + const streamClone = stream.clone() + streamClone.groupTitle = 'International' + groupedStreams.add(streamClone) + } + }) + + groupedStreams = groupedStreams.sortBy((stream: Stream) => { + if (stream.groupTitle === 'International') return 'ZZ' + if (stream.groupTitle === 'Undefined') return 'ZZZ' + + return stream.groupTitle + }) + + const playlist = new Playlist(groupedStreams, { public: true }) + const filepath = 'index.country.m3u' + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL + ) + } +} diff --git a/scripts/generators/indexGenerator.ts b/scripts/generators/indexGenerator.ts index 6dacf4bba1..fdfdde19d3 100644 --- a/scripts/generators/indexGenerator.ts +++ b/scripts/generators/indexGenerator.ts @@ -1,40 +1,45 @@ -import { Collection, File, Storage } from '@freearhey/core' -import { Stream, Playlist } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type IndexGeneratorProps = { - streams: Collection - logFile: File -} - -export class IndexGenerator implements Generator { - streams: Collection - storage: Storage - logFile: File - - constructor({ streams, logFile }: IndexGeneratorProps) { - this.streams = streams.clone() - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - const sfwStreams = this.streams - .orderBy(stream => stream.getTitle()) - .filter((stream: Stream) => stream.isSFW()) - .map((stream: Stream) => { - const groupTitle = stream.getCategoryNames().join(';') - if (groupTitle) stream.groupTitle = groupTitle - - return stream - }) - - const playlist = new Playlist(sfwStreams, { public: true }) - const filepath = 'index.m3u' - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL - ) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Stream, Playlist } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' + +type IndexGeneratorProps = { + streams: Collection + logFile: File +} + +export class IndexGenerator implements Generator { + streams: Collection + storage: Storage + logFile: File + + constructor({ streams, logFile }: IndexGeneratorProps) { + this.streams = streams.clone() + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate(): Promise { + const sfwStreams = this.streams + .sortBy(stream => stream.title) + .filter((stream: Stream) => stream.isSFW()) + .map((stream: Stream) => { + const groupTitle = stream + .getCategories() + .map(category => category.name) + .sort() + .join(';') + if (groupTitle) stream.groupTitle = groupTitle + + return stream + }) + + const playlist = new Playlist(sfwStreams, { public: true }) + const filepath = 'index.m3u' + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL + ) + } +} diff --git a/scripts/generators/indexLanguageGenerator.ts b/scripts/generators/indexLanguageGenerator.ts index 96fb17cee0..5e93a46adb 100644 --- a/scripts/generators/indexLanguageGenerator.ts +++ b/scripts/generators/indexLanguageGenerator.ts @@ -1,54 +1,57 @@ -import { Collection, Storage, File } from '@freearhey/core' -import { Stream, Playlist, Language } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type IndexLanguageGeneratorProps = { - streams: Collection - logFile: File -} - -export class IndexLanguageGenerator implements Generator { - streams: Collection - storage: Storage - logFile: File - - constructor({ streams, logFile }: IndexLanguageGeneratorProps) { - this.streams = streams.clone() - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - let groupedStreams = new Collection() - this.streams - .orderBy((stream: Stream) => stream.getTitle()) - .filter((stream: Stream) => stream.isSFW()) - .forEach((stream: Stream) => { - if (!stream.hasLanguages()) { - const streamClone = stream.clone() - streamClone.groupTitle = 'Undefined' - groupedStreams.add(streamClone) - return - } - - stream.getLanguages().forEach((language: Language) => { - const streamClone = stream.clone() - streamClone.groupTitle = language.name - groupedStreams.add(streamClone) - }) - }) - - groupedStreams = groupedStreams.orderBy((stream: Stream) => { - if (stream.groupTitle === 'Undefined') return 'ZZ' - return stream.groupTitle - }) - - const playlist = new Playlist(groupedStreams, { public: true }) - const filepath = 'index.language.m3u' - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL - ) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Stream, Playlist } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' +import * as sdk from '@iptv-org/sdk' + +type IndexLanguageGeneratorProps = { + streams: Collection + logFile: File +} + +export class IndexLanguageGenerator implements Generator { + streams: Collection + storage: Storage + logFile: File + + constructor({ streams, logFile }: IndexLanguageGeneratorProps) { + this.streams = streams.clone() + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate(): Promise { + let groupedStreams = new Collection() + this.streams + .sortBy((stream: Stream) => stream.title) + .filter((stream: Stream) => stream.isSFW()) + .forEach((stream: Stream) => { + const streamLanguages = stream.getLanguages() + if (streamLanguages.isEmpty()) { + const streamClone = stream.clone() + streamClone.groupTitle = 'Undefined' + groupedStreams.add(streamClone) + return + } + + streamLanguages.forEach((language: sdk.Models.Language) => { + const streamClone = stream.clone() + streamClone.groupTitle = language.name + groupedStreams.add(streamClone) + }) + }) + + groupedStreams = groupedStreams.sortBy((stream: Stream) => { + if (stream.groupTitle === 'Undefined') return 'ZZ' + return stream.groupTitle + }) + + const playlist = new Playlist(groupedStreams, { public: true }) + const filepath = 'index.language.m3u' + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL + ) + } +} diff --git a/scripts/generators/indexNsfwGenerator.ts b/scripts/generators/indexNsfwGenerator.ts deleted file mode 100644 index 14f3189d51..0000000000 --- a/scripts/generators/indexNsfwGenerator.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Collection, File, Storage } from '@freearhey/core' -import { Stream, Playlist } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type IndexNsfwGeneratorProps = { - streams: Collection - logFile: File -} - -export class IndexNsfwGenerator implements Generator { - streams: Collection - storage: Storage - logFile: File - - constructor({ streams, logFile }: IndexNsfwGeneratorProps) { - this.streams = streams.clone() - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - const allStreams = this.streams.orderBy((stream: Stream) => stream.getTitle()) - - const playlist = new Playlist(allStreams, { public: true }) - const filepath = 'index.nsfw.m3u' - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL - ) - } -} diff --git a/scripts/generators/languagesGenerator.ts b/scripts/generators/languagesGenerator.ts index f07e984d13..28fce5241a 100644 --- a/scripts/generators/languagesGenerator.ts +++ b/scripts/generators/languagesGenerator.ts @@ -1,57 +1,58 @@ -import { Collection, Storage, File } from '@freearhey/core' -import { Playlist, Language, Stream } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type LanguagesGeneratorProps = { streams: Collection; logFile: File } - -export class LanguagesGenerator implements Generator { - streams: Collection - storage: Storage - logFile: File - - constructor({ streams, logFile }: LanguagesGeneratorProps) { - this.streams = streams.clone() - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - const streams = this.streams - .orderBy((stream: Stream) => stream.getTitle()) - .filter((stream: Stream) => stream.isSFW()) - - let languages = new Collection() - streams.forEach((stream: Stream) => { - languages = languages.concat(stream.getLanguages()) - }) - - languages - .filter(Boolean) - .uniqBy((language: Language) => language.code) - .orderBy((language: Language) => language.name) - .forEach(async (language: Language) => { - const languageStreams = streams.filter((stream: Stream) => stream.hasLanguage(language)) - - if (languageStreams.isEmpty()) return - - const playlist = new Playlist(languageStreams, { public: true }) - const filepath = `languages/${language.code}.m3u` - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + EOL - ) - }) - - const undefinedStreams = streams.filter((stream: Stream) => !stream.hasLanguages()) - - if (undefinedStreams.isEmpty()) return - - const playlist = new Playlist(undefinedStreams, { public: true }) - const filepath = 'languages/undefined.m3u' - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + EOL - ) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Playlist, Stream } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' +import * as sdk from '@iptv-org/sdk' + +type LanguagesGeneratorProps = { streams: Collection; logFile: File } + +export class LanguagesGenerator implements Generator { + streams: Collection + storage: Storage + logFile: File + + constructor({ streams, logFile }: LanguagesGeneratorProps) { + this.streams = streams.clone() + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate(): Promise { + const streams: Collection = this.streams + .sortBy((stream: Stream) => stream.title) + .filter((stream: Stream) => stream.isSFW()) + + const languages = new Collection() + streams.forEach((stream: Stream) => { + languages.concat(stream.getLanguages()) + }) + + languages + .filter(Boolean) + .uniqBy((language: sdk.Models.Language) => language.code) + .sortBy((language: sdk.Models.Language) => language.name) + .forEach(async (language: sdk.Models.Language) => { + const languageStreams = streams.filter((stream: Stream) => stream.hasLanguage(language)) + + if (languageStreams.isEmpty()) return + + const playlist = new Playlist(languageStreams, { public: true }) + const filepath = `languages/${language.code}.m3u` + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + EOL + ) + }) + + const undefinedStreams = streams.filter((stream: Stream) => stream.getLanguages().isEmpty()) + if (undefinedStreams.isEmpty()) return + + const playlist = new Playlist(undefinedStreams, { public: true }) + const filepath = 'languages/undefined.m3u' + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + EOL + ) + } +} diff --git a/scripts/generators/rawGenerator.ts b/scripts/generators/rawGenerator.ts index 2816432d93..f16bdf76a7 100644 --- a/scripts/generators/rawGenerator.ts +++ b/scripts/generators/rawGenerator.ts @@ -1,40 +1,45 @@ -import { Collection, Storage, File } from '@freearhey/core' -import { Stream, Playlist } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type RawGeneratorProps = { - streams: Collection - logFile: File -} - -export class RawGenerator implements Generator { - streams: Collection - storage: Storage - logFile: File - - constructor({ streams, logFile }: RawGeneratorProps) { - this.streams = streams.clone() - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate() { - const files = this.streams.groupBy((stream: Stream) => stream.getFilename()) - - for (const filename of files.keys()) { - const streams = new Collection(files.get(filename)).map((stream: Stream) => { - const groupTitle = stream.getCategoryNames().join(';') - if (groupTitle) stream.groupTitle = groupTitle - - return stream - }) - const playlist = new Playlist(streams, { public: true }) - const filepath = `raw/${filename}` - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'raw', filepath, count: playlist.streams.count() }) + EOL - ) - } - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Stream, Playlist } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' + +type RawGeneratorProps = { + streams: Collection + logFile: File +} + +export class RawGenerator implements Generator { + streams: Collection + storage: Storage + logFile: File + + constructor({ streams, logFile }: RawGeneratorProps) { + this.streams = streams.clone() + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate() { + const files = this.streams.groupBy((stream: Stream) => stream.getFilename()) + + for (const filename of files.keys()) { + const streams = new Collection(files.get(filename)).map((stream: Stream) => { + const groupTitle = stream + .getCategories() + .map(category => category.name) + .sort() + .join(';') + if (groupTitle) stream.groupTitle = groupTitle + + return stream + }) + const playlist = new Playlist(streams, { public: true }) + const filepath = `raw/${filename}` + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'raw', filepath, count: playlist.streams.count() }) + EOL + ) + } + } +} diff --git a/scripts/generators/regionsGenerator.ts b/scripts/generators/regionsGenerator.ts index 02112974ed..c0d669bed6 100644 --- a/scripts/generators/regionsGenerator.ts +++ b/scripts/generators/regionsGenerator.ts @@ -1,41 +1,54 @@ -import { Collection, Storage, File } from '@freearhey/core' -import { Playlist, Region, Stream } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type RegionsGeneratorProps = { - streams: Collection - regions: Collection - logFile: File -} - -export class RegionsGenerator implements Generator { - streams: Collection - regions: Collection - storage: Storage - logFile: File - - constructor({ streams, regions, logFile }: RegionsGeneratorProps) { - this.streams = streams.clone() - this.regions = regions - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - const streams = this.streams - .orderBy((stream: Stream) => stream.getTitle()) - .filter((stream: Stream) => stream.isSFW()) - - this.regions.forEach(async (region: Region) => { - const regionStreams = streams.filter((stream: Stream) => stream.isBroadcastInRegion(region)) - - const playlist = new Playlist(regionStreams, { public: true }) - const filepath = `regions/${region.code.toLowerCase()}.m3u` - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() }) + EOL - ) - }) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Playlist, Stream } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' +import * as sdk from '@iptv-org/sdk' + +type RegionsGeneratorProps = { + streams: Collection + regions: Collection + logFile: File +} + +export class RegionsGenerator implements Generator { + streams: Collection + regions: Collection + storage: Storage + logFile: File + + constructor({ streams, regions, logFile }: RegionsGeneratorProps) { + this.streams = streams.clone() + this.regions = regions + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate(): Promise { + const streams = this.streams + .sortBy((stream: Stream) => stream.title) + .filter((stream: Stream) => stream.isSFW()) + + const streamsGroupedByRegionCode = {} + streams.forEach((stream: Stream) => { + stream.getBroadcastRegions().forEach((region: sdk.Models.Region) => { + if (streamsGroupedByRegionCode[region.code]) { + streamsGroupedByRegionCode[region.code].add(stream) + } else { + streamsGroupedByRegionCode[region.code] = new Collection([stream]) + } + }) + }) + + for (const regionCode in streamsGroupedByRegionCode) { + const regionStreams = streamsGroupedByRegionCode[regionCode] + + const playlist = new Playlist(regionStreams, { public: true }) + const filepath = `regions/${regionCode.toLowerCase()}.m3u` + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() }) + EOL + ) + } + } +} diff --git a/scripts/generators/sourcesGenerator.ts b/scripts/generators/sourcesGenerator.ts index 607c05b0e5..d90bc7f742 100644 --- a/scripts/generators/sourcesGenerator.ts +++ b/scripts/generators/sourcesGenerator.ts @@ -1,43 +1,49 @@ -import { Collection, Storage, File, type Dictionary } from '@freearhey/core' -import { Stream, Playlist } from '../models' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type SourcesGeneratorProps = { - streams: Collection - logFile: File -} - -export class SourcesGenerator implements Generator { - streams: Collection - storage: Storage - logFile: File - - constructor({ streams, logFile }: SourcesGeneratorProps) { - this.streams = streams.clone() - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate() { - const files: Dictionary = this.streams.groupBy((stream: Stream) => stream.getFilename()) - - for (const filename of files.keys()) { - if (!filename) continue - - let streams = new Collection(files.get(filename)) - streams = streams.map((stream: Stream) => { - const groupTitle = stream.getCategoryNames().join(';') - if (groupTitle) stream.groupTitle = groupTitle - - return stream - }) - const playlist = new Playlist(streams, { public: true }) - const filepath = `sources/${filename}` - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'source', filepath, count: playlist.streams.count() }) + EOL - ) - } - } -} +import { Collection, Dictionary } from '@freearhey/core' +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Stream, Playlist } from '../models' +import { Generator } from './generator' + +type SourcesGeneratorProps = { + streams: Collection + logFile: File +} + +export class SourcesGenerator implements Generator { + streams: Collection + storage: Storage + logFile: File + + constructor({ streams, logFile }: SourcesGeneratorProps) { + this.streams = streams.clone() + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate() { + const files: Dictionary = this.streams.groupBy((stream: Stream) => + stream.getFilename() + ) + + for (const filename of files.keys()) { + if (!filename) continue + + const streams = new Collection(files.get(filename)).map((stream: Stream) => { + const groupTitle = stream + .getCategories() + .map(category => category.name) + .sort() + .join(';') + if (groupTitle) stream.groupTitle = groupTitle + + return stream + }) + const playlist = new Playlist(streams, { public: true }) + const filepath = `sources/${filename}` + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'source', filepath, count: playlist.streams.count() }) + EOL + ) + } + } +} diff --git a/scripts/generators/subdivisionsGenerator.ts b/scripts/generators/subdivisionsGenerator.ts index 8ddce23328..758f6738f2 100644 --- a/scripts/generators/subdivisionsGenerator.ts +++ b/scripts/generators/subdivisionsGenerator.ts @@ -1,45 +1,54 @@ -import { Subdivision, Stream, Playlist } from '../models' -import { Collection, Storage, File } from '@freearhey/core' -import { PUBLIC_DIR, EOL } from '../constants' -import { Generator } from './generator' - -type SubdivisionsGeneratorProps = { - streams: Collection - subdivisions: Collection - logFile: File -} - -export class SubdivisionsGenerator implements Generator { - streams: Collection - subdivisions: Collection - storage: Storage - logFile: File - - constructor({ streams, subdivisions, logFile }: SubdivisionsGeneratorProps) { - this.streams = streams.clone() - this.subdivisions = subdivisions - this.storage = new Storage(PUBLIC_DIR) - this.logFile = logFile - } - - async generate(): Promise { - const streams = this.streams - .orderBy((stream: Stream) => stream.getTitle()) - .filter((stream: Stream) => stream.isSFW()) - - this.subdivisions.forEach(async (subdivision: Subdivision) => { - const subdivisionStreams = streams.filter((stream: Stream) => - stream.isBroadcastInSubdivision(subdivision) - ) - - if (subdivisionStreams.isEmpty()) return - - const playlist = new Playlist(subdivisionStreams, { public: true }) - const filepath = `subdivisions/${subdivision.code.toLowerCase()}.m3u` - await this.storage.save(filepath, playlist.toString()) - this.logFile.append( - JSON.stringify({ type: 'subdivision', filepath, count: playlist.streams.count() }) + EOL - ) - }) - } -} +import { Storage, File } from '@freearhey/storage-js' +import { PUBLIC_DIR, EOL } from '../constants' +import { Stream, Playlist } from '../models' +import { Collection } from '@freearhey/core' +import { Generator } from './generator' +import * as sdk from '@iptv-org/sdk' + +type SubdivisionsGeneratorProps = { + streams: Collection + subdivisions: Collection + logFile: File +} + +export class SubdivisionsGenerator implements Generator { + streams: Collection + subdivisions: Collection + storage: Storage + logFile: File + + constructor({ streams, subdivisions, logFile }: SubdivisionsGeneratorProps) { + this.streams = streams.clone() + this.subdivisions = subdivisions + this.storage = new Storage(PUBLIC_DIR) + this.logFile = logFile + } + + async generate(): Promise { + const streams = this.streams + .sortBy((stream: Stream) => stream.title) + .filter((stream: Stream) => stream.isSFW()) + + const streamsGroupedBySubdivisionCode = {} + streams.forEach((stream: Stream) => { + stream.getBroadcastSubdivisions().forEach((subdivision: sdk.Models.Subdivision) => { + if (streamsGroupedBySubdivisionCode[subdivision.code]) { + streamsGroupedBySubdivisionCode[subdivision.code].add(stream) + } else { + streamsGroupedBySubdivisionCode[subdivision.code] = new Collection([stream]) + } + }) + }) + + for (const subdivisionCode in streamsGroupedBySubdivisionCode) { + const subdivisionStreams = streamsGroupedBySubdivisionCode[subdivisionCode] + + const playlist = new Playlist(subdivisionStreams, { public: true }) + const filepath = `subdivisions/${subdivisionCode.toLowerCase()}.m3u` + await this.storage.save(filepath, playlist.toString()) + this.logFile.append( + JSON.stringify({ type: 'subdivision', filepath, count: playlist.streams.count() }) + EOL + ) + } + } +} diff --git a/scripts/models/blocklistRecord.ts b/scripts/models/blocklistRecord.ts deleted file mode 100644 index 632a1d4ddc..0000000000 --- a/scripts/models/blocklistRecord.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { BlocklistRecordData } from '../types/blocklistRecord' - -export class BlocklistRecord { - channelId: string - reason: string - ref: string - - constructor(data?: BlocklistRecordData) { - if (!data) return - - this.channelId = data.channel - this.reason = data.reason - this.ref = data.ref - } -} diff --git a/scripts/models/broadcastArea.ts b/scripts/models/broadcastArea.ts deleted file mode 100644 index da49fe5402..0000000000 --- a/scripts/models/broadcastArea.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { Collection, Dictionary } from '@freearhey/core' -import { City, Subdivision, Region, Country } from './' - -export class BroadcastArea { - codes: Collection - citiesIncluded: Collection - subdivisionsIncluded: Collection - countriesIncluded: Collection - regionsIncluded: Collection - - constructor(codes: Collection) { - this.codes = codes - } - - withLocations( - citiesKeyByCode: Dictionary, - subdivisionsKeyByCode: Dictionary, - countriesKeyByCode: Dictionary, - regionsKeyByCode: Dictionary - ): this { - const citiesIncluded = new Collection() - const subdivisionsIncluded = new Collection() - const countriesIncluded = new Collection() - let regionsIncluded = new Collection() - - this.codes.forEach((value: string) => { - const [type, code] = value.split('/') - - switch (type) { - case 'ct': { - const city: City = citiesKeyByCode.get(code) - if (!city) return - citiesIncluded.add(city) - if (city.subdivision) subdivisionsIncluded.add(city.subdivision) - if (city.subdivision && city.subdivision.parent) - subdivisionsIncluded.add(city.subdivision.parent) - if (city.country) countriesIncluded.add(city.country) - regionsIncluded = regionsIncluded.concat(city.getRegions()) - break - } - case 's': { - const subdivision: Subdivision = subdivisionsKeyByCode.get(code) - if (!subdivision) return - subdivisionsIncluded.add(subdivision) - if (subdivision.country) countriesIncluded.add(subdivision.country) - regionsIncluded = regionsIncluded.concat(subdivision.getRegions()) - break - } - case 'c': { - const country: Country = countriesKeyByCode.get(code) - if (!country) return - countriesIncluded.add(country) - regionsIncluded = regionsIncluded.concat(country.getRegions()) - break - } - case 'r': { - const region: Region = regionsKeyByCode.get(code) - if (!region) return - regionsIncluded = regionsIncluded.concat(region.getRegions()) - break - } - } - }) - - this.citiesIncluded = citiesIncluded.uniqBy((city: City) => city.code) - this.subdivisionsIncluded = subdivisionsIncluded.uniqBy( - (subdivision: Subdivision) => subdivision.code - ) - this.countriesIncluded = countriesIncluded.uniqBy((country: Country) => country.code) - this.regionsIncluded = regionsIncluded.uniqBy((region: Region) => region.code) - - return this - } - - getCountries(): Collection { - return this.countriesIncluded || new Collection() - } - - getSubdivisions(): Collection { - return this.subdivisionsIncluded || new Collection() - } - - getCities(): Collection { - return this.citiesIncluded || new Collection() - } - - getRegions(): Collection { - return this.regionsIncluded || new Collection() - } - - includesCountry(country: Country): boolean { - return this.getCountries().includes((_country: Country) => _country.code === country.code) - } - - includesSubdivision(subdivision: Subdivision): boolean { - return this.getSubdivisions().includes( - (_subdivision: Subdivision) => _subdivision.code === subdivision.code - ) - } - - includesRegion(region: Region): boolean { - return this.getRegions().includes((_region: Region) => _region.code === region.code) - } - - includesCity(city: City): boolean { - return this.getCities().includes((_city: City) => _city.code === city.code) - } -} diff --git a/scripts/models/category.ts b/scripts/models/category.ts deleted file mode 100644 index 5b228a86d0..0000000000 --- a/scripts/models/category.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { CategoryData, CategorySerializedData } from '../types/category' - -export class Category { - id: string - name: string - - constructor(data: CategoryData) { - this.id = data.id - this.name = data.name - } - - serialize(): CategorySerializedData { - return { - id: this.id, - name: this.name - } - } -} diff --git a/scripts/models/channel.ts b/scripts/models/channel.ts deleted file mode 100644 index 7a90146cf0..0000000000 --- a/scripts/models/channel.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { Collection, Dictionary } from '@freearhey/core' -import { Category, Country, Feed, Guide, Logo, Stream, Subdivision } from './index' -import type { ChannelData, ChannelSearchableData, ChannelSerializedData } from '../types/channel' - -export class Channel { - id: string - name: string - altNames: Collection - network?: string - owners: Collection - countryCode: string - country?: Country - subdivisionCode?: string - subdivision?: Subdivision - cityName?: string - categoryIds: Collection - categories: Collection = new Collection() - isNSFW: boolean - launched?: string - closed?: string - replacedBy?: string - isClosed: boolean - website?: string - feeds?: Collection - logos: Collection = new Collection() - - constructor(data?: ChannelData) { - if (!data) return - - this.id = data.id - this.name = data.name - this.altNames = new Collection(data.alt_names) - this.network = data.network || undefined - this.owners = new Collection(data.owners) - this.countryCode = data.country - this.subdivisionCode = data.subdivision || undefined - this.cityName = data.city || undefined - this.categoryIds = new Collection(data.categories) - this.isNSFW = data.is_nsfw - this.launched = data.launched || undefined - this.closed = data.closed || undefined - this.replacedBy = data.replaced_by || undefined - this.website = data.website || undefined - this.isClosed = !!data.closed || !!data.replaced_by - } - - withSubdivision(subdivisionsKeyByCode: Dictionary): this { - if (!this.subdivisionCode) return this - - this.subdivision = subdivisionsKeyByCode.get(this.subdivisionCode) - - return this - } - - withCountry(countriesKeyByCode: Dictionary): this { - this.country = countriesKeyByCode.get(this.countryCode) - - return this - } - - withCategories(categoriesKeyById: Dictionary): this { - this.categories = this.categoryIds - .map((id: string) => categoriesKeyById.get(id)) - .filter(Boolean) - - return this - } - - withFeeds(feedsGroupedByChannelId: Dictionary): this { - this.feeds = new Collection(feedsGroupedByChannelId.get(this.id)) - - return this - } - - withLogos(logosGroupedByChannelId: Dictionary): this { - if (this.id) this.logos = new Collection(logosGroupedByChannelId.get(this.id)) - - return this - } - - getCountry(): Country | undefined { - return this.country - } - - getSubdivision(): Subdivision | undefined { - return this.subdivision - } - - getCategories(): Collection { - return this.categories || new Collection() - } - - hasCategories(): boolean { - return !!this.categories && this.categories.notEmpty() - } - - hasCategory(category: Category): boolean { - return ( - !!this.categories && - this.categories.includes((_category: Category) => _category.id === category.id) - ) - } - - getFeeds(): Collection { - if (!this.feeds) return new Collection() - - return this.feeds - } - - getGuides(): Collection { - let guides = new Collection() - - this.getFeeds().forEach((feed: Feed) => { - guides = guides.concat(feed.getGuides()) - }) - - return guides - } - - getGuideNames(): Collection { - return this.getGuides() - .map((guide: Guide) => guide.siteName) - .uniq() - } - - getStreams(): Collection { - let streams = new Collection() - - this.getFeeds().forEach((feed: Feed) => { - streams = streams.concat(feed.getStreams()) - }) - - return streams - } - - getStreamTitles(): Collection { - return this.getStreams() - .map((stream: Stream) => stream.getTitle()) - .uniq() - } - - getFeedFullNames(): Collection { - return this.getFeeds() - .map((feed: Feed) => feed.getFullName()) - .uniq() - } - - isSFW(): boolean { - return this.isNSFW === false - } - - getLogos(): Collection { - function feed(logo: Logo): number { - if (!logo.feed) return 1 - if (logo.feed.isMain) return 1 - - return 0 - } - - function format(logo: Logo): number { - const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 } - - return logo.format ? levelByFormat[logo.format] : 0 - } - - function size(logo: Logo): number { - return Math.abs(512 - logo.width) + Math.abs(512 - logo.height) - } - - return this.logos.orderBy([feed, format, size], ['desc', 'desc', 'asc'], false) - } - - getLogo(): Logo | undefined { - return this.getLogos().first() - } - - hasLogo(): boolean { - return this.getLogos().notEmpty() - } - - getSearchable(): ChannelSearchableData { - return { - id: this.id, - name: this.name, - altNames: this.altNames.all(), - guideNames: this.getGuideNames().all(), - streamTitles: this.getStreamTitles().all(), - feedFullNames: this.getFeedFullNames().all() - } - } - - serialize(): ChannelSerializedData { - return { - id: this.id, - name: this.name, - altNames: this.altNames.all(), - network: this.network, - owners: this.owners.all(), - countryCode: this.countryCode, - country: this.country ? this.country.serialize() : undefined, - subdivisionCode: this.subdivisionCode, - subdivision: this.subdivision ? this.subdivision.serialize() : undefined, - cityName: this.cityName, - categoryIds: this.categoryIds.all(), - categories: this.categories.map((category: Category) => category.serialize()).all(), - isNSFW: this.isNSFW, - launched: this.launched, - closed: this.closed, - replacedBy: this.replacedBy, - website: this.website - } - } - - deserialize(data: ChannelSerializedData): this { - this.id = data.id - this.name = data.name - this.altNames = new Collection(data.altNames) - this.network = data.network - this.owners = new Collection(data.owners) - this.countryCode = data.countryCode - this.country = data.country ? new Country().deserialize(data.country) : undefined - this.subdivisionCode = data.subdivisionCode - this.cityName = data.cityName - this.categoryIds = new Collection(data.categoryIds) - this.isNSFW = data.isNSFW - this.launched = data.launched - this.closed = data.closed - this.replacedBy = data.replacedBy - this.website = data.website - - return this - } -} diff --git a/scripts/models/city.ts b/scripts/models/city.ts deleted file mode 100644 index 6ce9173ac9..0000000000 --- a/scripts/models/city.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { Collection, Dictionary } from '@freearhey/core' -import { Country, Region, Subdivision } from '.' -import type { CityData, CitySerializedData } from '../types/city' - -export class City { - code: string - name: string - countryCode: string - country?: Country - subdivisionCode?: string - subdivision?: Subdivision - wikidataId: string - regions?: Collection - - constructor(data?: CityData) { - if (!data) return - - this.code = data.code - this.name = data.name - this.countryCode = data.country - this.subdivisionCode = data.subdivision || undefined - this.wikidataId = data.wikidata_id - } - - withCountry(countriesKeyByCode: Dictionary): this { - this.country = countriesKeyByCode.get(this.countryCode) - - return this - } - - withSubdivision(subdivisionsKeyByCode: Dictionary): this { - if (!this.subdivisionCode) return this - - this.subdivision = subdivisionsKeyByCode.get(this.subdivisionCode) - - return this - } - - withRegions(regions: Collection): this { - this.regions = regions.filter((region: Region) => - region.countryCodes.includes(this.countryCode) - ) - - return this - } - - getRegions(): Collection { - if (!this.regions) return new Collection() - - return this.regions - } - - serialize(): CitySerializedData { - return { - code: this.code, - name: this.name, - countryCode: this.countryCode, - country: this.country ? this.country.serialize() : undefined, - subdivisionCode: this.subdivisionCode || null, - subdivision: this.subdivision ? this.subdivision.serialize() : undefined, - wikidataId: this.wikidataId - } - } - - deserialize(data: CitySerializedData): this { - this.code = data.code - this.name = data.name - this.countryCode = data.countryCode - this.country = data.country ? new Country().deserialize(data.country) : undefined - this.subdivisionCode = data.subdivisionCode || undefined - this.subdivision = data.subdivision - ? new Subdivision().deserialize(data.subdivision) - : undefined - this.wikidataId = data.wikidataId - - return this - } -} diff --git a/scripts/models/country.ts b/scripts/models/country.ts deleted file mode 100644 index b9699f7235..0000000000 --- a/scripts/models/country.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { Collection, Dictionary } from '@freearhey/core' -import { Region, Language, Subdivision } from '.' -import type { CountryData, CountrySerializedData } from '../types/country' -import { SubdivisionSerializedData } from '../types/subdivision' -import { RegionSerializedData } from '../types/region' - -export class Country { - code: string - name: string - flag: string - languageCode: string - language?: Language - subdivisions?: Collection - regions?: Collection - cities?: Collection - - constructor(data?: CountryData) { - if (!data) return - - this.code = data.code - this.name = data.name - this.flag = data.flag - this.languageCode = data.lang - } - - withSubdivisions(subdivisionsGroupedByCountryCode: Dictionary): this { - this.subdivisions = new Collection(subdivisionsGroupedByCountryCode.get(this.code)) - - return this - } - - withRegions(regions: Collection): this { - this.regions = regions.filter((region: Region) => region.includesCountryCode(this.code)) - - return this - } - - withCities(citiesGroupedByCountryCode: Dictionary): this { - this.cities = new Collection(citiesGroupedByCountryCode.get(this.code)) - - return this - } - - withLanguage(languagesKeyByCode: Dictionary): this { - this.language = languagesKeyByCode.get(this.languageCode) - - return this - } - - getLanguage(): Language | undefined { - return this.language - } - - getRegions(): Collection { - return this.regions || new Collection() - } - - getSubdivisions(): Collection { - return this.subdivisions || new Collection() - } - - getCities(): Collection { - return this.cities || new Collection() - } - - serialize(): CountrySerializedData { - return { - code: this.code, - name: this.name, - flag: this.flag, - languageCode: this.languageCode, - language: this.language ? this.language.serialize() : null, - subdivisions: this.subdivisions - ? this.subdivisions.map((subdivision: Subdivision) => subdivision.serialize()).all() - : [], - regions: this.regions ? this.regions.map((region: Region) => region.serialize()).all() : [] - } - } - - deserialize(data: CountrySerializedData): this { - this.code = data.code - this.name = data.name - this.flag = data.flag - this.languageCode = data.languageCode - this.language = data.language ? new Language().deserialize(data.language) : undefined - this.subdivisions = new Collection(data.subdivisions).map((data: SubdivisionSerializedData) => - new Subdivision().deserialize(data) - ) - this.regions = new Collection(data.regions).map((data: RegionSerializedData) => - new Region().deserialize(data) - ) - - return this - } -} diff --git a/scripts/models/feed.ts b/scripts/models/feed.ts deleted file mode 100644 index a6713e265a..0000000000 --- a/scripts/models/feed.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { Country, Language, Region, Channel, Subdivision, BroadcastArea, City } from './index' -import { Collection, Dictionary } from '@freearhey/core' -import type { FeedData } from '../types/feed' - -export class Feed { - channelId: string - channel?: Channel - id: string - name: string - isMain: boolean - broadcastAreaCodes: Collection - broadcastArea?: BroadcastArea - languageCodes: Collection - languages?: Collection - timezoneIds: Collection - timezones?: Collection - videoFormat: string - guides?: Collection - streams?: Collection - - constructor(data: FeedData) { - this.channelId = data.channel - this.id = data.id - this.name = data.name - this.isMain = data.is_main - this.broadcastAreaCodes = new Collection(data.broadcast_area) - this.languageCodes = new Collection(data.languages) - this.timezoneIds = new Collection(data.timezones) - this.videoFormat = data.video_format - } - - withChannel(channelsKeyById: Dictionary): this { - this.channel = channelsKeyById.get(this.channelId) - - return this - } - - withStreams(streamsGroupedById: Dictionary): this { - this.streams = new Collection(streamsGroupedById.get(`${this.channelId}@${this.id}`)) - - if (this.isMain) { - this.streams = this.streams.concat(new Collection(streamsGroupedById.get(this.channelId))) - } - - return this - } - - withGuides(guidesGroupedByStreamId: Dictionary): this { - this.guides = new Collection(guidesGroupedByStreamId.get(`${this.channelId}@${this.id}`)) - - if (this.isMain) { - this.guides = this.guides.concat(new Collection(guidesGroupedByStreamId.get(this.channelId))) - } - - return this - } - - withLanguages(languagesKeyByCode: Dictionary): this { - this.languages = this.languageCodes - .map((code: string) => languagesKeyByCode.get(code)) - .filter(Boolean) - - return this - } - - withTimezones(timezonesKeyById: Dictionary): this { - this.timezones = this.timezoneIds.map((id: string) => timezonesKeyById.get(id)).filter(Boolean) - - return this - } - - withBroadcastArea( - citiesKeyByCode: Dictionary, - subdivisionsKeyByCode: Dictionary, - countriesKeyByCode: Dictionary, - regionsKeyByCode: Dictionary - ): this { - this.broadcastArea = new BroadcastArea(this.broadcastAreaCodes).withLocations( - citiesKeyByCode, - subdivisionsKeyByCode, - countriesKeyByCode, - regionsKeyByCode - ) - - return this - } - - hasBroadcastArea(): boolean { - return !!this.broadcastArea - } - - getBroadcastCountries(): Collection { - if (!this.broadcastArea) return new Collection() - - return this.broadcastArea.getCountries() - } - - getBroadcastRegions(): Collection { - if (!this.broadcastArea) return new Collection() - - return this.broadcastArea.getRegions() - } - - getTimezones(): Collection { - return this.timezones || new Collection() - } - - getLanguages(): Collection { - return this.languages || new Collection() - } - - hasLanguages(): boolean { - return !!this.languages && this.languages.notEmpty() - } - - hasLanguage(language: Language): boolean { - return ( - !!this.languages && - this.languages.includes((_language: Language) => _language.code === language.code) - ) - } - - isBroadcastInCity(city: City): boolean { - if (!this.broadcastArea) return false - - return this.broadcastArea.includesCity(city) - } - - isBroadcastInSubdivision(subdivision: Subdivision): boolean { - if (!this.broadcastArea) return false - - return this.broadcastArea.includesSubdivision(subdivision) - } - - isBroadcastInCountry(country: Country): boolean { - if (!this.broadcastArea) return false - - return this.broadcastArea.includesCountry(country) - } - - isBroadcastInRegion(region: Region): boolean { - if (!this.broadcastArea) return false - - return this.broadcastArea.includesRegion(region) - } - - isInternational(): boolean { - if (!this.broadcastArea) return false - - return this.broadcastArea.codes.join(',').includes('r/') - } - - getGuides(): Collection { - if (!this.guides) return new Collection() - - return this.guides - } - - getStreams(): Collection { - if (!this.streams) return new Collection() - - return this.streams - } - - getFullName(): string { - if (!this.channel) return '' - - return `${this.channel.name} ${this.name}` - } -} diff --git a/scripts/models/guide.ts b/scripts/models/guide.ts deleted file mode 100644 index 3bc849d86e..0000000000 --- a/scripts/models/guide.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { GuideData, GuideSerializedData } from '../types/guide' - -export class Guide { - channelId?: string - feedId?: string - siteDomain: string - siteId: string - siteName: string - languageCode: string - - constructor(data?: GuideData) { - if (!data) return - - this.channelId = data.channel - this.feedId = data.feed - this.siteDomain = data.site - this.siteId = data.site_id - this.siteName = data.site_name - this.languageCode = data.lang - } - - getUUID(): string { - return this.getStreamId() + this.siteId - } - - getStreamId(): string | undefined { - if (!this.channelId) return undefined - if (!this.feedId) return this.channelId - - return `${this.channelId}@${this.feedId}` - } - - serialize(): GuideSerializedData { - return { - channelId: this.channelId, - feedId: this.feedId, - siteDomain: this.siteDomain, - siteId: this.siteId, - siteName: this.siteName, - languageCode: this.languageCode - } - } - - deserialize(data: GuideSerializedData): this { - this.channelId = data.channelId - this.feedId = data.feedId - this.siteDomain = data.siteDomain - this.siteId = data.siteId - this.siteName = data.siteName - this.languageCode = data.languageCode - - return this - } -} diff --git a/scripts/models/index.ts b/scripts/models/index.ts index e8fe346289..f4b06f6dd5 100644 --- a/scripts/models/index.ts +++ b/scripts/models/index.ts @@ -1,16 +1,3 @@ -export * from './blocklistRecord' -export * from './broadcastArea' -export * from './category' -export * from './channel' -export * from './city' -export * from './country' -export * from './feed' -export * from './guide' -export * from './issue' -export * from './language' -export * from './logo' -export * from './playlist' -export * from './region' -export * from './stream' -export * from './subdivision' -export * from './timezone' +export * from './issue' +export * from './playlist' +export * from './stream' diff --git a/scripts/models/language.ts b/scripts/models/language.ts deleted file mode 100644 index 1e6df829bc..0000000000 --- a/scripts/models/language.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { LanguageData, LanguageSerializedData } from '../types/language' - -export class Language { - code: string - name: string - - constructor(data?: LanguageData) { - if (!data) return - - this.code = data.code - this.name = data.name - } - - serialize(): LanguageSerializedData { - return { - code: this.code, - name: this.name - } - } - - deserialize(data: LanguageSerializedData): this { - this.code = data.code - this.name = data.name - - return this - } -} diff --git a/scripts/models/logo.ts b/scripts/models/logo.ts deleted file mode 100644 index 3cc85fb9da..0000000000 --- a/scripts/models/logo.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Collection, type Dictionary } from '@freearhey/core' -import type { LogoData } from '../types/logo' -import { type Feed } from './feed' - -export class Logo { - channelId: string - feedId?: string - feed: Feed - tags: Collection - width: number - height: number - format?: string - url: string - - constructor(data?: LogoData) { - if (!data) return - - this.channelId = data.channel - this.feedId = data.feed || undefined - this.tags = new Collection(data.tags) - this.width = data.width - this.height = data.height - this.format = data.format || undefined - this.url = data.url - } - - withFeed(feedsKeyById: Dictionary): this { - if (!this.feedId) return this - - this.feed = feedsKeyById.get(this.feedId) - - return this - } - - getStreamId(): string { - if (!this.feedId) return this.channelId - - return `${this.channelId}@${this.feedId}` - } -} diff --git a/scripts/models/playlist.ts b/scripts/models/playlist.ts index d3022a5af6..f6a2b25d60 100644 --- a/scripts/models/playlist.ts +++ b/scripts/models/playlist.ts @@ -1,28 +1,28 @@ -import { Collection } from '@freearhey/core' -import { Stream } from '../models' - -type PlaylistOptions = { - public: boolean -} - -export class Playlist { - streams: Collection - options: { - public: boolean - } - - constructor(streams: Collection, options?: PlaylistOptions) { - this.streams = streams - this.options = options || { public: false } - } - - toString() { - let output = '#EXTM3U\r\n' - - this.streams.forEach((stream: Stream) => { - output += stream.toString(this.options) + '\r\n' - }) - - return output - } -} +import { Collection } from '@freearhey/core' +import { Stream } from '../models' + +type PlaylistOptions = { + public: boolean +} + +export class Playlist { + streams: Collection + options: { + public: boolean + } + + constructor(streams: Collection, options?: PlaylistOptions) { + this.streams = streams + this.options = options || { public: false } + } + + toString() { + let output = '#EXTM3U\r\n' + + this.streams.forEach((stream: Stream) => { + output += stream.toString(this.options) + '\r\n' + }) + + return output + } +} diff --git a/scripts/models/region.ts b/scripts/models/region.ts deleted file mode 100644 index 5fe52ad5a9..0000000000 --- a/scripts/models/region.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { Collection, Dictionary } from '@freearhey/core' -import { City, Country, Subdivision } from '.' -import type { RegionData, RegionSerializedData } from '../types/region' -import { CountrySerializedData } from '../types/country' -import { SubdivisionSerializedData } from '../types/subdivision' -import { CitySerializedData } from '../types/city' - -export class Region { - code: string - name: string - countryCodes: Collection - countries?: Collection - subdivisions?: Collection - cities?: Collection - regions?: Collection - - constructor(data?: RegionData) { - if (!data) return - - this.code = data.code - this.name = data.name - this.countryCodes = new Collection(data.countries) - } - - withCountries(countriesKeyByCode: Dictionary): this { - this.countries = this.countryCodes.map((code: string) => countriesKeyByCode.get(code)) - - return this - } - - withSubdivisions(subdivisions: Collection): this { - this.subdivisions = subdivisions.filter( - (subdivision: Subdivision) => this.countryCodes.indexOf(subdivision.countryCode) > -1 - ) - - return this - } - - withCities(cities: Collection): this { - this.cities = cities.filter((city: City) => this.countryCodes.indexOf(city.countryCode) > -1) - - return this - } - - withRegions(regions: Collection): this { - this.regions = regions.filter( - (region: Region) => !region.countryCodes.intersects(this.countryCodes).isEmpty() - ) - - return this - } - - getSubdivisions(): Collection { - if (!this.subdivisions) return new Collection() - - return this.subdivisions - } - - getCountries(): Collection { - if (!this.countries) return new Collection() - - return this.countries - } - - getCities(): Collection { - if (!this.cities) return new Collection() - - return this.cities - } - - getRegions(): Collection { - if (!this.regions) return new Collection() - - return this.regions - } - - includesCountryCode(code: string): boolean { - return this.countryCodes.includes((countryCode: string) => countryCode === code) - } - - isWorldwide(): boolean { - return ['INT', 'WW'].includes(this.code) - } - - serialize(): RegionSerializedData { - return { - code: this.code, - name: this.name, - countryCodes: this.countryCodes.all(), - countries: this.getCountries() - .map((country: Country) => country.serialize()) - .all(), - subdivisions: this.getSubdivisions() - .map((subdivision: Subdivision) => subdivision.serialize()) - .all(), - cities: this.getCities() - .map((city: City) => city.serialize()) - .all() - } - } - - deserialize(data: RegionSerializedData): this { - this.code = data.code - this.name = data.name - this.countryCodes = new Collection(data.countryCodes) - this.countries = new Collection(data.countries).map((data: CountrySerializedData) => - new Country().deserialize(data) - ) - this.subdivisions = new Collection(data.subdivisions).map((data: SubdivisionSerializedData) => - new Subdivision().deserialize(data) - ) - this.cities = new Collection(data.cities).map((data: CitySerializedData) => - new City().deserialize(data) - ) - - return this - } -} diff --git a/scripts/models/stream.ts b/scripts/models/stream.ts index a465081ea6..c050930b4d 100644 --- a/scripts/models/stream.ts +++ b/scripts/models/stream.ts @@ -1,474 +1,461 @@ -import { - Feed, - Channel, - Category, - Region, - Subdivision, - Country, - Language, - Logo, - City -} from './index' -import { URL, Collection, Dictionary } from '@freearhey/core' -import type { StreamData } from '../types/stream' -import parser from 'iptv-playlist-parser' -import { IssueData } from '../core' -import path from 'node:path' - -export class Stream { - title: string - url: string - id?: string - channelId?: string - channel?: Channel - feedId?: string - feed?: Feed - logos: Collection = new Collection() - filepath?: string - line?: number - label?: string - verticalResolution?: number - isInterlaced?: boolean - referrer?: string - userAgent?: string - groupTitle: string = 'Undefined' - removed: boolean = false - directives: Collection = new Collection() - - constructor(data?: StreamData) { - if (!data) return - - const id = - data.channelId && data.feedId ? [data.channelId, data.feedId].join('@') : data.channelId - const { verticalResolution, isInterlaced } = parseQuality(data.quality) - - this.id = id || undefined - this.channelId = data.channelId || undefined - this.feedId = data.feedId || undefined - this.title = data.title || '' - this.url = data.url - this.referrer = data.referrer || undefined - this.userAgent = data.userAgent || undefined - this.verticalResolution = verticalResolution || undefined - this.isInterlaced = isInterlaced || undefined - this.label = data.label || undefined - this.directives = new Collection(data.directives) - } - - update(issueData: IssueData): this { - const data = { - label: issueData.getString('label'), - quality: issueData.getString('quality'), - httpUserAgent: issueData.getString('httpUserAgent'), - httpReferrer: issueData.getString('httpReferrer'), - newStreamUrl: issueData.getString('newStreamUrl'), - directives: issueData.getArray('directives') - } - - if (data.label !== undefined) this.label = data.label - if (data.quality !== undefined) this.setQuality(data.quality) - if (data.httpUserAgent !== undefined) this.userAgent = data.httpUserAgent - if (data.httpReferrer !== undefined) this.referrer = data.httpReferrer - if (data.newStreamUrl !== undefined) this.url = data.newStreamUrl - if (data.directives !== undefined) this.directives = new Collection(data.directives) - - return this - } - - fromPlaylistItem(data: parser.PlaylistItem): this { - function parseName(name: string): { - title: string - label: string - quality: string - } { - let title = name - const [, label] = title.match(/ \[(.*)\]$/) || [null, ''] - title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '') - const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, ''] - title = title.replace(new RegExp(` \\(${quality}\\)$`), '') - - return { title, label, quality } - } - - function parseDirectives(string: string) { - const directives = new Collection() - - if (!string) return directives - - const supportedDirectives = ['#EXTVLCOPT', '#KODIPROP'] - const lines = string.split('\r\n') - const regex = new RegExp(`^${supportedDirectives.join('|')}`, 'i') - - lines.forEach((line: string) => { - if (regex.test(line)) { - directives.add(line.trim()) - } - }) - - return directives - } - - if (!data.name) throw new Error('"name" property is required') - if (!data.url) throw new Error('"url" property is required') - - const [channelId, feedId] = data.tvg.id.split('@') - const { title, label, quality } = parseName(data.name) - const { verticalResolution, isInterlaced } = parseQuality(quality) - - this.id = data.tvg.id || undefined - this.feedId = feedId || undefined - this.channelId = channelId || undefined - this.line = data.line - this.label = label || undefined - this.title = title - this.verticalResolution = verticalResolution || undefined - this.isInterlaced = isInterlaced || undefined - this.url = data.url - this.referrer = data.http.referrer || undefined - this.userAgent = data.http['user-agent'] || undefined - this.directives = parseDirectives(data.raw) - - return this - } - - withChannel(channelsKeyById: Dictionary): this { - if (!this.channelId) return this - - this.channel = channelsKeyById.get(this.channelId) - - return this - } - - withFeed(feedsGroupedByChannelId: Dictionary): this { - if (!this.channelId) return this - - const channelFeeds = feedsGroupedByChannelId.get(this.channelId) || [] - if (this.feedId) this.feed = channelFeeds.find((feed: Feed) => feed.id === this.feedId) - if (!this.feedId && !this.feed) this.feed = channelFeeds.find((feed: Feed) => feed.isMain) - - return this - } - - withLogos(logosGroupedByStreamId: Dictionary): this { - if (this.id) this.logos = new Collection(logosGroupedByStreamId.get(this.id)) - - return this - } - - setId(id: string): this { - this.id = id - - return this - } - - setChannelId(channelId: string): this { - this.channelId = channelId - - return this - } - - setFeedId(feedId: string | undefined): this { - this.feedId = feedId - - return this - } - - setQuality(quality: string): this { - const { verticalResolution, isInterlaced } = parseQuality(quality) - - this.verticalResolution = verticalResolution || undefined - this.isInterlaced = isInterlaced || undefined - - return this - } - - getLine(): number { - return this.line || -1 - } - - getFilename(): string { - if (!this.filepath) return '' - - return path.basename(this.filepath) - } - - setFilepath(filepath: string): this { - this.filepath = filepath - - return this - } - - updateFilepath(): this { - if (!this.channel) return this - - this.filepath = `${this.channel.countryCode.toLowerCase()}.m3u` - - return this - } - - getChannelId(): string { - return this.channelId || '' - } - - getFeedId(): string { - if (this.feedId) return this.feedId - if (this.feed) return this.feed.id - return '' - } - - getFilepath(): string { - return this.filepath || '' - } - - getReferrer(): string { - return this.referrer || '' - } - - getUserAgent(): string { - return this.userAgent || '' - } - - getQuality(): string { - if (!this.verticalResolution) return '' - - let quality = this.verticalResolution.toString() - - if (this.isInterlaced) quality += 'i' - else quality += 'p' - - return quality - } - - hasId(): boolean { - return !!this.id - } - - hasQuality(): boolean { - return !!this.verticalResolution - } - - getVerticalResolution(): number { - if (!this.hasQuality()) return 0 - - return parseInt(this.getQuality().replace(/p|i/, '')) - } - - updateTitle(): this { - if (!this.channel) return this - - this.title = this.channel.name - if (this.feed && !this.feed.isMain) { - this.title += ` ${this.feed.name}` - } - - return this - } - - updateId(): this { - if (!this.channel) return this - if (this.feed) { - this.id = `${this.channel.id}@${this.feed.id}` - } else { - this.id = this.channel.id - } - - return this - } - - normalizeURL() { - const url = new URL(this.url) - - this.url = url.normalize().toString() - } - - clone(): Stream { - return Object.assign(Object.create(Object.getPrototypeOf(this)), this) - } - - hasChannel() { - return !!this.channel - } - - getBroadcastRegions(): Collection { - return this.feed ? this.feed.getBroadcastRegions() : new Collection() - } - - getBroadcastCountries(): Collection { - return this.feed ? this.feed.getBroadcastCountries() : new Collection() - } - - hasBroadcastArea(): boolean { - return this.feed ? this.feed.hasBroadcastArea() : false - } - - isSFW(): boolean { - return this.channel ? this.channel.isSFW() : true - } - - hasCategories(): boolean { - return this.channel ? this.channel.hasCategories() : false - } - - hasCategory(category: Category): boolean { - return this.channel ? this.channel.hasCategory(category) : false - } - - getCategoryNames(): string[] { - return this.getCategories() - .map((category: Category) => category.name) - .sort() - .all() - } - - getCategories(): Collection { - return this.channel ? this.channel.getCategories() : new Collection() - } - - getLanguages(): Collection { - return this.feed ? this.feed.getLanguages() : new Collection() - } - - hasLanguages() { - return this.feed ? this.feed.hasLanguages() : false - } - - hasLanguage(language: Language) { - return this.feed ? this.feed.hasLanguage(language) : false - } - - getBroadcastAreaCodes(): Collection { - return this.feed ? this.feed.broadcastAreaCodes : new Collection() - } - - isBroadcastInCity(city: City): boolean { - return this.feed ? this.feed.isBroadcastInCity(city) : false - } - - isBroadcastInSubdivision(subdivision: Subdivision): boolean { - return this.feed ? this.feed.isBroadcastInSubdivision(subdivision) : false - } - - isBroadcastInCountry(country: Country): boolean { - return this.feed ? this.feed.isBroadcastInCountry(country) : false - } - - isBroadcastInRegion(region: Region): boolean { - return this.feed ? this.feed.isBroadcastInRegion(region) : false - } - - isInternational(): boolean { - return this.feed ? this.feed.isInternational() : false - } - - getLogos(): Collection { - function format(logo: Logo): number { - const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 } - - return logo.format ? levelByFormat[logo.format] : 0 - } - - function size(logo: Logo): number { - return Math.abs(512 - logo.width) + Math.abs(512 - logo.height) - } - - return this.logos.orderBy([format, size], ['desc', 'asc'], false) - } - - getLogo(): Logo | undefined { - return this.getLogos().first() - } - - hasLogo(): boolean { - return this.getLogos().notEmpty() - } - - getLogoUrl(): string { - let logo: Logo | undefined - - if (this.hasLogo()) logo = this.getLogo() - else logo = this?.channel?.getLogo() - - return logo ? logo.url : '' - } - - getTitle(): string { - return this.title || '' - } - - getFullTitle(): string { - let title = `${this.getTitle()}` - - if (this.getQuality()) { - title += ` (${this.getQuality()})` - } - - if (this.label) { - title += ` [${this.label}]` - } - - return title - } - - getLabel(): string { - return this.label || '' - } - - getId(): string { - return this.id || '' - } - - toJSON() { - return { - channel: this.channelId || null, - feed: this.feedId || null, - title: this.title, - url: this.url, - referrer: this.referrer || null, - user_agent: this.userAgent || null, - quality: this.getQuality() || null - } - } - - toString(options: { public: boolean }) { - let output = `#EXTINF:-1 tvg-id="${this.getId()}"` - - if (options.public) { - output += ` tvg-logo="${this.getLogoUrl()}" group-title="${this.groupTitle}"` - } - - if (this.referrer) { - output += ` http-referrer="${this.referrer}"` - } - - if (this.userAgent) { - output += ` http-user-agent="${this.userAgent}"` - } - - output += `,${this.getFullTitle()}` - - this.directives.forEach((prop: string) => { - output += `\r\n${prop}` - }) - - output += `\r\n${this.url}` - - return output - } -} - -function escapeRegExp(text) { - return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&') -} - -function parseQuality(quality: string | null): { - verticalResolution: number | null - isInterlaced: boolean | null -} { - if (!quality) return { verticalResolution: null, isInterlaced: null } - const [, verticalResolutionString] = quality.match(/^(\d+)/) || [null, undefined] - const isInterlaced = /i$/i.test(quality) - let verticalResolution = 0 - if (verticalResolutionString) verticalResolution = parseInt(verticalResolutionString) - - return { verticalResolution, isInterlaced } -} +import { Collection } from '@freearhey/core' +import parser from 'iptv-playlist-parser' +import { normalizeURL } from '../utils' +import * as sdk from '@iptv-org/sdk' +import { IssueData } from '../core' +import { data } from '../api' +import path from 'node:path' + +export class Stream extends sdk.Models.Stream { + directives: Collection + filepath?: string + line?: number + groupTitle: string = 'Undefined' + removed: boolean = false + tvgId?: string + label: string | null + + updateWithIssue(issueData: IssueData): this { + const data = { + label: issueData.getString('label'), + quality: issueData.getString('quality'), + httpUserAgent: issueData.getString('httpUserAgent'), + httpReferrer: issueData.getString('httpReferrer'), + newStreamUrl: issueData.getString('newStreamUrl'), + directives: issueData.getArray('directives') + } + + if (data.label !== undefined) this.label = data.label + if (data.quality !== undefined) this.quality = data.quality + if (data.httpUserAgent !== undefined) this.user_agent = data.httpUserAgent + if (data.httpReferrer !== undefined) this.referrer = data.httpReferrer + if (data.newStreamUrl !== undefined) this.url = data.newStreamUrl + if (data.directives !== undefined) this.setDirectives(data.directives) + + return this + } + + static fromPlaylistItem(data: parser.PlaylistItem): Stream { + function escapeRegExp(text) { + return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&') + } + + function parseName(name: string): { + title: string + label: string + quality: string + } { + let title = name + const [, label] = title.match(/ \[(.*)\]$/) || [null, ''] + title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '') + const [, quality] = title.match(/ \(([0-9]+[p|i])\)$/) || [null, ''] + title = title.replace(new RegExp(` \\(${quality}\\)$`), '') + + return { title, label, quality } + } + + function parseDirectives(string: string): Collection { + const directives = new Collection() + + if (!string) return directives + + const supportedDirectives = ['#EXTVLCOPT', '#KODIPROP'] + const lines = string.split('\r\n') + const regex = new RegExp(`^${supportedDirectives.join('|')}`, 'i') + + lines.forEach((line: string) => { + if (regex.test(line)) { + directives.add(line.trim()) + } + }) + + return directives + } + + if (!data.name) throw new Error('"name" property is required') + if (!data.url) throw new Error('"url" property is required') + + const [channelId, feedId] = data.tvg.id.split('@') + const { title, label, quality } = parseName(data.name) + + const stream = new Stream({ + channel: channelId || null, + feed: feedId || null, + title: title, + quality: quality || null, + url: data.url, + referrer: data.http.referrer || null, + user_agent: data.http['user-agent'] || null + }) + + stream.tvgId = data.tvg.id + stream.line = data.line + stream.label = label || null + stream.directives = parseDirectives(data.raw) + + return stream + } + + isSFW(): boolean { + const channel = this.getChannel() + + if (!channel) return true + + return !channel.is_nsfw + } + + getUniqKey(): string { + const filepath = this.getFilepath() + const tvgId = this.getTvgId() + + return filepath + tvgId + this.url + } + + getVerticalResolution(): number { + if (!this.quality) return 0 + + const [, verticalResolutionString] = this.quality.match(/^(\d+)/) || ['', '0'] + + return parseInt(verticalResolutionString) + } + + getBroadcastCountries(): Collection { + const countries = new Collection() + + const feed = this.getFeed() + if (!feed) return countries + + feed + .getBroadcastArea() + .getLocations() + .forEach((location: sdk.Models.BroadcastAreaLocation) => { + let country: sdk.Models.Country | undefined + switch (location.type) { + case 'country': { + country = data.countriesKeyByCode.get(location.code) + break + } + case 'subdivision': { + const subdivision = data.subdivisionsKeyByCode.get(location.code) + if (!subdivision) break + country = data.countriesKeyByCode.get(subdivision.country) + break + } + case 'city': { + const city = data.citiesKeyByCode.get(location.code) + if (!city) break + country = data.countriesKeyByCode.get(city.country) + break + } + } + + if (country) countries.add(country) + }) + + return countries.uniqBy((country: sdk.Models.Country) => country.code) + } + + getBroadcastSubdivisions(): Collection { + const subdivisions = new Collection() + + const feed = this.getFeed() + if (!feed) return subdivisions + + feed + .getBroadcastArea() + .getLocations() + .forEach((location: sdk.Models.BroadcastAreaLocation) => { + switch (location.type) { + case 'subdivision': { + const subdivision = data.subdivisionsKeyByCode.get(location.code) + if (!subdivision) break + subdivisions.add(subdivision) + if (!subdivision.parent) break + const parentSubdivision = data.subdivisionsKeyByCode.get(subdivision.parent) + if (!parentSubdivision) break + subdivisions.add(parentSubdivision) + break + } + case 'city': { + const city = data.citiesKeyByCode.get(location.code) + if (!city || !city.subdivision) break + const subdivision = data.subdivisionsKeyByCode.get(city.subdivision) + if (!subdivision) break + subdivisions.add(subdivision) + if (!subdivision.parent) break + const parentSubdivision = data.subdivisionsKeyByCode.get(subdivision.parent) + if (!parentSubdivision) break + subdivisions.add(parentSubdivision) + break + } + } + }) + + return subdivisions.uniqBy((subdivision: sdk.Models.Subdivision) => subdivision.code) + } + + getBroadcastCities(): Collection { + const cities = new Collection() + + const feed = this.getFeed() + if (!feed) return cities + + feed + .getBroadcastArea() + .getLocations() + .forEach((location: sdk.Models.BroadcastAreaLocation) => { + if (location.type !== 'city') return + + const city = data.citiesKeyByCode.get(location.code) + + if (city) cities.add(city) + }) + + return cities.uniqBy((city: sdk.Models.City) => city.code) + } + + getBroadcastRegions(): Collection { + const regions = new Collection() + + const feed = this.getFeed() + if (!feed) return regions + + feed + .getBroadcastArea() + .getLocations() + .forEach((location: sdk.Models.BroadcastAreaLocation) => { + switch (location.type) { + case 'region': { + const region = data.regionsKeyByCode.get(location.code) + if (!region) break + regions.add(region) + + const relatedRegions = data.regions.filter((_region: sdk.Models.Region) => + new Collection(_region.countries) + .intersects(new Collection(region.countries)) + .isNotEmpty() + ) + regions.concat(relatedRegions) + break + } + case 'country': { + const country = data.countriesKeyByCode.get(location.code) + if (!country) break + const countryRegions = data.regions.filter((_region: sdk.Models.Region) => + new Collection(_region.countries).includes( + (code: string) => code === country.code + ) + ) + regions.concat(countryRegions) + break + } + case 'subdivision': { + const subdivision = data.subdivisionsKeyByCode.get(location.code) + if (!subdivision) break + const subdivisionRegions = data.regions.filter((_region: sdk.Models.Region) => + new Collection(_region.countries).includes( + (code: string) => code === subdivision.country + ) + ) + regions.concat(subdivisionRegions) + break + } + case 'city': { + const city = data.citiesKeyByCode.get(location.code) + if (!city) break + const cityRegions = data.regions.filter((_region: sdk.Models.Region) => + new Collection(_region.countries).includes( + (code: string) => code === city.country + ) + ) + regions.concat(cityRegions) + break + } + } + }) + + return regions.uniqBy((region: sdk.Models.Region) => region.code) + } + + isInternational(): boolean { + const feed = this.getFeed() + if (!feed) return false + + const broadcastAreaCodes = feed.getBroadcastArea().codes + if (broadcastAreaCodes.join(';').includes('r/')) return true + if (broadcastAreaCodes.filter(code => code.includes('c/')).length > 1) return true + + return false + } + + hasCategory(category: sdk.Models.Category): boolean { + const channel = this.getChannel() + + if (!channel) return false + + const found = channel.categories.find((id: string) => id === category.id) + + return !!found + } + + hasLanguage(language: sdk.Models.Language): boolean { + const found = this.getLanguages().find( + (_language: sdk.Models.Language) => _language.code === language.code + ) + + return !!found + } + + setDirectives(directives: string[]): this { + this.directives = new Collection(directives).filter((directive: string) => + /^(#KODIPROP|#EXTVLCOPT)/.test(directive) + ) + + return this + } + + updateTvgId(): this { + if (!this.channel) return this + if (this.feed) { + this.tvgId = `${this.channel}@${this.feed}` + } else { + this.tvgId = this.channel + } + + return this + } + + updateFilepath(): this { + const channel = this.getChannel() + if (!channel) return this + + this.filepath = `${channel.country.toLowerCase()}.m3u` + + return this + } + + updateTitle(): this { + const channel = this.getChannel() + + if (!channel) return this + + const feed = this.getFeed() + + this.title = channel.name + if (feed && !feed.is_main) { + this.title += ` ${feed.name}` + } + + return this + } + + normalizeURL() { + this.url = normalizeURL(this.url) + } + + getLogos(): Collection { + const logos = super.getLogos() + + if (logos.isEmpty()) return new Collection() + + function format(logo: sdk.Models.Logo): number { + const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 } + + return logo.format ? levelByFormat[logo.format] : 0 + } + + function size(logo: sdk.Models.Logo): number { + return Math.abs(512 - logo.width) + Math.abs(512 - logo.height) + } + + return logos.sortBy([format, size], ['desc', 'asc'], false) + } + + getFilepath(): string { + return this.filepath || '' + } + + getFilename(): string { + return path.basename(this.getFilepath()) + } + + getLine(): number { + return this.line || -1 + } + + getTvgId(): string { + if (this.tvgId) return this.tvgId + + return this.getId() + } + + getTvgLogo(): string { + const logo = this.getLogos().first() + + return logo ? logo.url : '' + } + + getFullTitle(): string { + let title = `${this.title}` + + if (this.quality) { + title += ` (${this.quality})` + } + + if (this.label) { + title += ` [${this.label}]` + } + + return title + } + + toString(options: { public?: boolean } = {}) { + options = { ...{ public: false }, ...options } + + let output = `#EXTINF:-1 tvg-id="${this.getTvgId()}"` + + if (options.public) { + output += ` tvg-logo="${this.getTvgLogo()}" group-title="${this.groupTitle}"` + } + + if (this.referrer) { + output += ` http-referrer="${this.referrer}"` + } + + if (this.user_agent) { + output += ` http-user-agent="${this.user_agent}"` + } + + output += `,${this.getFullTitle()}` + + this.directives.forEach((prop: string) => { + output += `\r\n${prop}` + }) + + output += `\r\n${this.url}` + + return output + } + + toObject(): sdk.Types.StreamData { + let feedId = this.feed + if (!feedId) { + const feed = this.getFeed() + if (feed) feedId = feed.id + } + + return { + channel: this.channel, + feed: feedId, + title: this.title, + url: this.url, + quality: this.quality, + user_agent: this.user_agent, + referrer: this.referrer + } + } + + clone(): Stream { + return Object.assign(Object.create(Object.getPrototypeOf(this)), this) + } +} diff --git a/scripts/models/subdivision.ts b/scripts/models/subdivision.ts deleted file mode 100644 index b70bd4ac5b..0000000000 --- a/scripts/models/subdivision.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { SubdivisionData, SubdivisionSerializedData } from '../types/subdivision' -import { Dictionary, Collection } from '@freearhey/core' -import { Country, Region } from '.' - -export class Subdivision { - code: string - name: string - countryCode: string - country?: Country - parentCode?: string - parent?: Subdivision - regions?: Collection - cities?: Collection - - constructor(data?: SubdivisionData) { - if (!data) return - - this.code = data.code - this.name = data.name - this.countryCode = data.country - this.parentCode = data.parent || undefined - } - - withCountry(countriesKeyByCode: Dictionary): this { - this.country = countriesKeyByCode.get(this.countryCode) - - return this - } - - withRegions(regions: Collection): this { - this.regions = regions.filter((region: Region) => - region.countryCodes.includes(this.countryCode) - ) - - return this - } - - withCities(citiesGroupedBySubdivisionCode: Dictionary): this { - this.cities = new Collection(citiesGroupedBySubdivisionCode.get(this.code)) - - return this - } - - withParent(subdivisionsKeyByCode: Dictionary): this { - if (!this.parentCode) return this - - this.parent = subdivisionsKeyByCode.get(this.parentCode) - - return this - } - - getRegions(): Collection { - if (!this.regions) return new Collection() - - return this.regions - } - - getCities(): Collection { - if (!this.cities) return new Collection() - - return this.cities - } - - serialize(): SubdivisionSerializedData { - return { - code: this.code, - name: this.name, - countryCode: this.countryCode, - country: this.country ? this.country.serialize() : undefined, - parentCode: this.parentCode || null - } - } - - deserialize(data: SubdivisionSerializedData): this { - this.code = data.code - this.name = data.name - this.countryCode = data.countryCode - this.country = data.country ? new Country().deserialize(data.country) : undefined - this.parentCode = data.parentCode || undefined - - return this - } -} diff --git a/scripts/models/timezone.ts b/scripts/models/timezone.ts deleted file mode 100644 index e4071138fe..0000000000 --- a/scripts/models/timezone.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { Collection, Dictionary } from '@freearhey/core' - -type TimezoneData = { - id: string - utc_offset: string - countries: string[] -} - -export class Timezone { - id: string - utcOffset: string - countryCodes: Collection - countries?: Collection - - constructor(data: TimezoneData) { - this.id = data.id - this.utcOffset = data.utc_offset - this.countryCodes = new Collection(data.countries) - } - - withCountries(countriesKeyByCode: Dictionary): this { - this.countries = this.countryCodes.map((code: string) => countriesKeyByCode.get(code)) - - return this - } - - getCountries(): Collection { - return this.countries || new Collection() - } -} diff --git a/scripts/tables/categoriesTable.ts b/scripts/tables/categoriesTable.ts index 0b763e4162..ca1432d56e 100644 --- a/scripts/tables/categoriesTable.ts +++ b/scripts/tables/categoriesTable.ts @@ -1,56 +1,63 @@ -import { Storage, Collection, File, Dictionary } from '@freearhey/core' -import { HTMLTable, LogParser, LogItem } from '../core' -import { LOGS_DIR, README_DIR } from '../constants' -import { Category } from '../models' -import { Table } from './table' - -type CategoriesTableProps = { - categoriesKeyById: Dictionary -} - -export class CategoriesTable implements Table { - categoriesKeyById: Dictionary - - constructor({ categoriesKeyById }: CategoriesTableProps) { - this.categoriesKeyById = categoriesKeyById - } - - async make() { - const parser = new LogParser() - const logsStorage = new Storage(LOGS_DIR) - const generatorsLog = await logsStorage.load('generators.log') - - let items = new Collection() - parser - .parse(generatorsLog) - .filter((logItem: LogItem) => logItem.type === 'category') - .forEach((logItem: LogItem) => { - const file = new File(logItem.filepath) - const categoryId = file.name() - const category: Category = this.categoriesKeyById.get(categoryId) - - items.add([ - category ? category.name : 'ZZ', - category ? category.name : 'Undefined', - logItem.count, - `https://iptv-org.github.io/iptv/${logItem.filepath}` - ]) - }) - - items = items - .orderBy(item => item[0]) - .map(item => { - item.shift() - return item - }) - - const table = new HTMLTable(items.all(), [ - { name: 'Category' }, - { name: 'Channels', align: 'right' }, - { name: 'Playlist', nowrap: true } - ]) - - const readmeStorage = new Storage(README_DIR) - await readmeStorage.save('_categories.md', table.toString()) - } -} +import { HTMLTable, HTMLTableItem, LogParser, LogItem, HTMLTableColumn } from '../core' +import { Storage, File } from '@freearhey/storage-js' +import { LOGS_DIR, README_DIR } from '../constants' +import { Collection } from '@freearhey/core' +import * as sdk from '@iptv-org/sdk' +import { Table } from './table' +import { data } from '../api' + +export class CategoriesTable implements Table { + async create() { + const parser = new LogParser() + const logsStorage = new Storage(LOGS_DIR) + const generatorsLog = await logsStorage.load('generators.log') + + let items = new Collection() + parser + .parse(generatorsLog) + .filter((logItem: LogItem) => logItem.type === 'category') + .forEach((logItem: LogItem) => { + if (logItem.filepath.includes('undefined')) { + items.add([ + 'ZZ', + 'Undefined', + logItem.count.toString(), + `https://iptv-org.github.io/iptv/${logItem.filepath}` + ]) + + return + } + + const file = new File(logItem.filepath) + const categoryId = file.name() + const category: sdk.Models.Category | undefined = data.categoriesKeyById.get(categoryId) + + if (!category) return + + items.add([ + category.name, + category.name, + logItem.count.toString(), + `https://iptv-org.github.io/iptv/${logItem.filepath}` + ]) + }) + + items = items + .sortBy(item => item[0]) + .map(item => { + item.shift() + return item + }) + + const columns = new Collection([ + { name: 'Category' }, + { name: 'Channels', align: 'right' }, + { name: 'Playlist', nowrap: true } + ]) + + const table = new HTMLTable(items, columns) + + const readmeStorage = new Storage(README_DIR) + await readmeStorage.save('_categories.md', table.toString()) + } +} diff --git a/scripts/tables/countriesTable.ts b/scripts/tables/countriesTable.ts index 8c9d2173a4..95914f6d30 100644 --- a/scripts/tables/countriesTable.ts +++ b/scripts/tables/countriesTable.ts @@ -1,189 +1,176 @@ -import { Storage, Collection, Dictionary } from '@freearhey/core' -import { City, Country, Subdivision } from '../models' -import { LOGS_DIR, README_DIR } from '../constants' -import { LogParser, LogItem } from '../core' -import { Table } from './table' - -type CountriesTableProps = { - countriesKeyByCode: Dictionary - subdivisionsKeyByCode: Dictionary - countries: Collection - subdivisions: Collection - cities: Collection -} - -export class CountriesTable implements Table { - countriesKeyByCode: Dictionary - subdivisionsKeyByCode: Dictionary - countries: Collection - subdivisions: Collection - cities: Collection - - constructor({ - countriesKeyByCode, - subdivisionsKeyByCode, - countries, - subdivisions, - cities - }: CountriesTableProps) { - this.countriesKeyByCode = countriesKeyByCode - this.subdivisionsKeyByCode = subdivisionsKeyByCode - this.countries = countries - this.subdivisions = subdivisions - this.cities = cities - } - - async make() { - const parser = new LogParser() - const logsStorage = new Storage(LOGS_DIR) - const generatorsLog = await logsStorage.load('generators.log') - const parsed = parser.parse(generatorsLog) - const logCountries = parsed.filter((logItem: LogItem) => logItem.type === 'country') - const logSubdivisions = parsed.filter((logItem: LogItem) => logItem.type === 'subdivision') - const logCities = parsed.filter((logItem: LogItem) => logItem.type === 'city') - - let items = new Collection() - this.countries.forEach((country: Country) => { - const countriesLogItem = logCountries.find( - (logItem: LogItem) => logItem.filepath === `countries/${country.code.toLowerCase()}.m3u` - ) - - const countryItem = { - index: country.name, - count: 0, - link: `https://iptv-org.github.io/iptv/countries/${country.code.toLowerCase()}.m3u`, - name: `${country.flag} ${country.name}`, - children: new Collection() - } - - if (countriesLogItem) { - countryItem.count = countriesLogItem.count - } - - const countrySubdivisions = this.subdivisions.filter( - (subdivision: Subdivision) => subdivision.countryCode === country.code - ) - const countryCities = this.cities.filter((city: City) => city.countryCode === country.code) - if (countrySubdivisions.notEmpty()) { - this.subdivisions.forEach((subdivision: Subdivision) => { - if (subdivision.countryCode !== country.code) return - const subdivisionCities = countryCities.filter( - (city: City) => - (city.subdivisionCode && city.subdivisionCode === subdivision.code) || - city.countryCode === subdivision.countryCode - ) - const subdivisionsLogItem = logSubdivisions.find( - (logItem: LogItem) => - logItem.filepath === `subdivisions/${subdivision.code.toLowerCase()}.m3u` - ) - - const subdivisionItem = { - index: subdivision.name, - name: subdivision.name, - count: 0, - link: `https://iptv-org.github.io/iptv/subdivisions/${subdivision.code.toLowerCase()}.m3u`, - children: new Collection() - } - - if (subdivisionsLogItem) { - subdivisionItem.count = subdivisionsLogItem.count - } - - subdivisionCities.forEach((city: City) => { - if (city.countryCode !== country.code || city.subdivisionCode !== subdivision.code) - return - const citiesLogItem = logCities.find( - (logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u` - ) - - if (!citiesLogItem) return - - subdivisionItem.children.add({ - index: city.name, - name: city.name, - count: citiesLogItem.count, - link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}` - }) - }) - - if (subdivisionItem.count > 0 || subdivisionItem.children.notEmpty()) { - countryItem.children.add(subdivisionItem) - } - }) - } else if (countryCities.notEmpty()) { - countryCities.forEach((city: City) => { - const citiesLogItem = logCities.find( - (logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u` - ) - - if (!citiesLogItem) return - - countryItem.children.add({ - index: city.name, - name: city.name, - count: citiesLogItem.count, - link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}`, - children: new Collection() - }) - }) - } - - if (countryItem.count > 0 || countryItem.children.notEmpty()) { - items.add(countryItem) - } - }) - - const internationalLogItem = logCountries.find( - (logItem: LogItem) => logItem.filepath === 'countries/int.m3u' - ) - - if (internationalLogItem) { - items.push({ - index: 'ZZ', - name: '🌐 International', - count: internationalLogItem.count, - link: `https://iptv-org.github.io/iptv/${internationalLogItem.filepath}`, - children: new Collection() - }) - } - - const undefinedLogItem = logCountries.find( - (logItem: LogItem) => logItem.filepath === 'countries/undefined.m3u' - ) - - if (undefinedLogItem) { - items.push({ - index: 'ZZZ', - name: 'Undefined', - count: undefinedLogItem.count, - link: `https://iptv-org.github.io/iptv/${undefinedLogItem.filepath}`, - children: new Collection() - }) - } - - items = items.orderBy(item => item.index) - - const output = items - .map(item => { - let row = `- ${item.name} ${item.link}` - - item.children - .orderBy(item => item.index) - .forEach(item => { - row += `\r\n - ${item.name} ${item.link}` - - item.children - .orderBy(item => item.index) - .forEach(item => { - row += `\r\n - ${item.name} ${item.link}` - }) - }) - - return row - }) - .join('\r\n') - - const readmeStorage = new Storage(README_DIR) - await readmeStorage.save('_countries.md', output) - } -} +import { LOGS_DIR, README_DIR } from '../constants' +import { Storage } from '@freearhey/storage-js' +import { Collection } from '@freearhey/core' +import { LogParser, LogItem } from '../core' +import * as sdk from '@iptv-org/sdk' +import { Table } from './table' +import { data } from '../api' + +type ListItem = { + index: string + count: number + link: string + name: string + children: Collection +} + +export class CountriesTable implements Table { + async create() { + const parser = new LogParser() + const logsStorage = new Storage(LOGS_DIR) + const generatorsLog = await logsStorage.load('generators.log') + const parsed = parser.parse(generatorsLog) + const logCountries = parsed.filter((logItem: LogItem) => logItem.type === 'country') + const logSubdivisions = parsed.filter((logItem: LogItem) => logItem.type === 'subdivision') + const logCities = parsed.filter((logItem: LogItem) => logItem.type === 'city') + + let items = new Collection() + data.countries.forEach((country: sdk.Models.Country) => { + const countryCode = country.code + const countriesLogItem = logCountries.find( + (logItem: LogItem) => logItem.filepath === `countries/${countryCode.toLowerCase()}.m3u` + ) + + const countryItem: ListItem = { + index: country.name, + count: 0, + link: `https://iptv-org.github.io/iptv/countries/${countryCode.toLowerCase()}.m3u`, + name: `${country.flag} ${country.name}`, + children: new Collection() + } + + if (countriesLogItem) { + countryItem.count = countriesLogItem.count + } + + const countrySubdivisions = data.subdivisions.filter( + (subdivision: sdk.Models.Subdivision) => subdivision.country === countryCode + ) + const countryCities = data.cities.filter( + (city: sdk.Models.City) => city.country === countryCode + ) + if (countrySubdivisions.isNotEmpty()) { + data.subdivisions.forEach((subdivision: sdk.Models.Subdivision) => { + if (subdivision.country !== countryCode) return + + const subdivisionCode = subdivision.code + const subdivisionCities = countryCities.filter( + (city: sdk.Models.City) => + (city.subdivision && city.subdivision === subdivisionCode) || + city.country === subdivision.country + ) + const subdivisionsLogItem = logSubdivisions.find( + (logItem: LogItem) => + logItem.filepath === `subdivisions/${subdivisionCode.toLowerCase()}.m3u` + ) + + const subdivisionItem: ListItem = { + index: subdivision.name, + name: subdivision.name, + count: 0, + link: `https://iptv-org.github.io/iptv/subdivisions/${subdivisionCode.toLowerCase()}.m3u`, + children: new Collection() + } + + if (subdivisionsLogItem) { + subdivisionItem.count = subdivisionsLogItem.count + } + + subdivisionCities.forEach((city: sdk.Models.City) => { + if (city.country !== countryCode || city.subdivision !== subdivisionCode) return + const citiesLogItem = logCities.find( + (logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u` + ) + + if (!citiesLogItem) return + + subdivisionItem.children.add({ + index: city.name, + name: city.name, + count: citiesLogItem.count, + link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}`, + children: new Collection() + }) + }) + + if (subdivisionItem.count > 0 || subdivisionItem.children.isNotEmpty()) { + countryItem.children.add(subdivisionItem) + } + }) + } else if (countryCities.isNotEmpty()) { + countryCities.forEach((city: sdk.Models.City) => { + const citiesLogItem = logCities.find( + (logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u` + ) + + if (!citiesLogItem) return + + countryItem.children.add({ + index: city.name, + name: city.name, + count: citiesLogItem.count, + link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}`, + children: new Collection() + }) + }) + } + + if (countryItem.count > 0 || countryItem.children.isNotEmpty()) { + items.add(countryItem) + } + }) + + const internationalLogItem = logCountries.find( + (logItem: LogItem) => logItem.filepath === 'countries/int.m3u' + ) + + if (internationalLogItem) { + items.add({ + index: 'ZZ', + name: '🌐 International', + count: internationalLogItem.count, + link: `https://iptv-org.github.io/iptv/${internationalLogItem.filepath}`, + children: new Collection() + }) + } + + const undefinedLogItem = logCountries.find( + (logItem: LogItem) => logItem.filepath === 'countries/undefined.m3u' + ) + + if (undefinedLogItem) { + items.add({ + index: 'ZZZ', + name: 'Undefined', + count: undefinedLogItem.count, + link: `https://iptv-org.github.io/iptv/${undefinedLogItem.filepath}`, + children: new Collection() + }) + } + + items = items.sortBy(item => item.index) + + const output = items + .map((item: ListItem) => { + let row = `- ${item.name} ${item.link}` + + item.children + .sortBy((item: ListItem) => item.index) + .forEach((item: ListItem) => { + row += `\r\n - ${item.name} ${item.link}` + + item.children + .sortBy((item: ListItem) => item.index) + .forEach((item: ListItem) => { + row += `\r\n - ${item.name} ${item.link}` + }) + }) + + return row + }) + .join('\r\n') + + const readmeStorage = new Storage(README_DIR) + await readmeStorage.save('_countries.md', output) + } +} diff --git a/scripts/tables/languagesTable.ts b/scripts/tables/languagesTable.ts index 7621907453..469166fade 100644 --- a/scripts/tables/languagesTable.ts +++ b/scripts/tables/languagesTable.ts @@ -1,56 +1,63 @@ -import { Storage, Collection, File, Dictionary } from '@freearhey/core' -import { HTMLTable, LogParser, LogItem } from '../core' -import { LOGS_DIR, README_DIR } from '../constants' -import { Language } from '../models' -import { Table } from './table' - -type LanguagesTableProps = { - languagesKeyByCode: Dictionary -} - -export class LanguagesTable implements Table { - languagesKeyByCode: Dictionary - - constructor({ languagesKeyByCode }: LanguagesTableProps) { - this.languagesKeyByCode = languagesKeyByCode - } - - async make() { - const parser = new LogParser() - const logsStorage = new Storage(LOGS_DIR) - const generatorsLog = await logsStorage.load('generators.log') - - let data = new Collection() - parser - .parse(generatorsLog) - .filter((logItem: LogItem) => logItem.type === 'language') - .forEach((logItem: LogItem) => { - const file = new File(logItem.filepath) - const languageCode = file.name() - const language: Language = this.languagesKeyByCode.get(languageCode) - - data.add([ - language ? language.name : 'ZZ', - language ? language.name : 'Undefined', - logItem.count, - `https://iptv-org.github.io/iptv/${logItem.filepath}` - ]) - }) - - data = data - .orderBy(item => item[0]) - .map(item => { - item.shift() - return item - }) - - const table = new HTMLTable(data.all(), [ - { name: 'Language', align: 'left' }, - { name: 'Channels', align: 'right' }, - { name: 'Playlist', align: 'left', nowrap: true } - ]) - - const readmeStorage = new Storage(README_DIR) - await readmeStorage.save('_languages.md', table.toString()) - } -} +import { HTMLTable, LogParser, LogItem, HTMLTableColumn, HTMLTableItem } from '../core' +import { Storage, File } from '@freearhey/storage-js' +import { LOGS_DIR, README_DIR } from '../constants' +import { Collection } from '@freearhey/core' +import * as sdk from '@iptv-org/sdk' +import { Table } from './table' +import { data } from '../api' + +export class LanguagesTable implements Table { + async create() { + const parser = new LogParser() + const logsStorage = new Storage(LOGS_DIR) + const generatorsLog = await logsStorage.load('generators.log') + + let items = new Collection() + parser + .parse(generatorsLog) + .filter((logItem: LogItem) => logItem.type === 'language') + .forEach((logItem: LogItem) => { + if (logItem.filepath.includes('undefined')) { + items.add([ + 'ZZ', + 'Undefined', + logItem.count.toString(), + `https://iptv-org.github.io/iptv/${logItem.filepath}` + ]) + + return + } + + const file = new File(logItem.filepath) + const languageCode = file.name() + const language: sdk.Models.Language | undefined = data.languagesKeyByCode.get(languageCode) + + if (!language) return + + items.add([ + language.name, + language.name, + logItem.count.toString(), + `https://iptv-org.github.io/iptv/${logItem.filepath}` + ]) + }) + + items = items + .sortBy(item => item[0]) + .map(item => { + item.shift() + return item + }) + + const columns = new Collection([ + { name: 'Language', align: 'left' }, + { name: 'Channels', align: 'right' }, + { name: 'Playlist', align: 'left', nowrap: true } + ]) + + const table = new HTMLTable(items, columns) + + const readmeStorage = new Storage(README_DIR) + await readmeStorage.save('_languages.md', table.toString()) + } +} diff --git a/scripts/tables/regionsTable.ts b/scripts/tables/regionsTable.ts index 2bf4b74f4f..39c58d98b7 100644 --- a/scripts/tables/regionsTable.ts +++ b/scripts/tables/regionsTable.ts @@ -1,52 +1,49 @@ -import { Storage, Collection } from '@freearhey/core' -import { LogParser, LogItem } from '../core' -import { LOGS_DIR, README_DIR } from '../constants' -import { Region } from '../models' -import { Table } from './table' - -type RegionsTableProps = { - regions: Collection -} - -export class RegionsTable implements Table { - regions: Collection - - constructor({ regions }: RegionsTableProps) { - this.regions = regions - } - - async make() { - const parser = new LogParser() - const logsStorage = new Storage(LOGS_DIR) - const generatorsLog = await logsStorage.load('generators.log') - const parsed = parser.parse(generatorsLog) - const logRegions = parsed.filter((logItem: LogItem) => logItem.type === 'region') - - let items = new Collection() - this.regions.forEach((region: Region) => { - const logItem = logRegions.find( - (logItem: LogItem) => logItem.filepath === `regions/${region.code.toLowerCase()}.m3u` - ) - - if (!logItem) return - - items.add({ - index: region.name, - name: region.name, - count: logItem.count, - link: `https://iptv-org.github.io/iptv/${logItem.filepath}` - }) - }) - - items = items.orderBy(item => item.index) - - const output = items - .map(item => { - return `- ${item.name} ${item.link}` - }) - .join('\r\n') - - const readmeStorage = new Storage(README_DIR) - await readmeStorage.save('_regions.md', output) - } -} +import { LOGS_DIR, README_DIR } from '../constants' +import { Storage } from '@freearhey/storage-js' +import { LogParser, LogItem } from '../core' +import { Collection } from '@freearhey/core' +import * as sdk from '@iptv-org/sdk' +import { Table } from './table' +import { data } from '../api' + +type ListItem = { + name: string + count: number + link: string +} + +export class RegionsTable implements Table { + async create() { + const parser = new LogParser() + const logsStorage = new Storage(LOGS_DIR) + const generatorsLog = await logsStorage.load('generators.log') + const parsed = parser.parse(generatorsLog) + const logRegions = parsed.filter((logItem: LogItem) => logItem.type === 'region') + + let items = new Collection() + data.regions.forEach((region: sdk.Models.Region) => { + const logItem = logRegions.find( + (logItem: LogItem) => logItem.filepath === `regions/${region.code.toLowerCase()}.m3u` + ) + + if (!logItem) return + + items.add({ + name: region.name, + count: logItem.count, + link: `https://iptv-org.github.io/iptv/${logItem.filepath}` + }) + }) + + items = items.sortBy(item => item.name) + + const output = items + .map(item => { + return `- ${item.name} ${item.link}` + }) + .join('\r\n') + + const readmeStorage = new Storage(README_DIR) + await readmeStorage.save('_regions.md', output) + } +} diff --git a/scripts/tables/table.ts b/scripts/tables/table.ts index b8bd21bd7e..b342b5e7e8 100644 --- a/scripts/tables/table.ts +++ b/scripts/tables/table.ts @@ -1,3 +1,3 @@ -export interface Table { - make(): void -} +export interface Table { + create(): void +} diff --git a/scripts/types/blocklistRecord.d.ts b/scripts/types/blocklistRecord.d.ts deleted file mode 100644 index 4b1d9e7dc8..0000000000 --- a/scripts/types/blocklistRecord.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type BlocklistRecordData = { - channel: string - reason: string - ref: string -} diff --git a/scripts/types/category.d.ts b/scripts/types/category.d.ts deleted file mode 100644 index e78d6c62ed..0000000000 --- a/scripts/types/category.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -export type CategorySerializedData = { - id: string - name: string -} - -export type CategoryData = { - id: string - name: string -} diff --git a/scripts/types/channel.d.ts b/scripts/types/channel.d.ts deleted file mode 100644 index 814fc53a79..0000000000 --- a/scripts/types/channel.d.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { Collection } from '@freearhey/core' -import type { CountrySerializedData } from './country' -import type { SubdivisionSerializedData } from './subdivision' -import type { CategorySerializedData } from './category' - -export type ChannelSerializedData = { - id: string - name: string - altNames: string[] - network?: string - owners: string[] - countryCode: string - country?: CountrySerializedData - subdivisionCode?: string - subdivision?: SubdivisionSerializedData - cityName?: string - categoryIds: string[] - categories?: CategorySerializedData[] - isNSFW: boolean - launched?: string - closed?: string - replacedBy?: string - website?: string -} - -export type ChannelData = { - id: string - name: string - alt_names: string[] - network: string - owners: Collection - country: string - subdivision: string - city: string - categories: Collection - is_nsfw: boolean - launched: string - closed: string - replaced_by: string - website: string -} - -export type ChannelSearchableData = { - id: string - name: string - altNames: string[] - guideNames: string[] - streamTitles: string[] - feedFullNames: string[] -} diff --git a/scripts/types/city.d.ts b/scripts/types/city.d.ts deleted file mode 100644 index 5c33ba5a9e..0000000000 --- a/scripts/types/city.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { CountrySerializedData } from './country' -import { SubdivisionSerializedData } from './subdivision' - -export type CitySerializedData = { - code: string - name: string - countryCode: string - country?: CountrySerializedData - subdivisionCode: string | null - subdivision?: SubdivisionSerializedData - wikidataId: string -} - -export type CityData = { - code: string - name: string - country: string - subdivision: string | null - wikidata_id: string -} diff --git a/scripts/types/country.d.ts b/scripts/types/country.d.ts deleted file mode 100644 index 9554d4c68c..0000000000 --- a/scripts/types/country.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { LanguageSerializedData } from './language' -import type { SubdivisionSerializedData } from './subdivision' -import type { RegionSerializedData } from './region' - -export type CountrySerializedData = { - code: string - name: string - flag: string - languageCode: string - language: LanguageSerializedData | null - subdivisions: SubdivisionSerializedData[] - regions: RegionSerializedData[] -} - -export type CountryData = { - code: string - name: string - lang: string - flag: string -} diff --git a/scripts/types/dataLoader.d.ts b/scripts/types/dataLoader.d.ts deleted file mode 100644 index 708361de99..0000000000 --- a/scripts/types/dataLoader.d.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Storage } from '@freearhey/core' - -export type DataLoaderProps = { - storage: Storage -} - -export type DataLoaderData = { - countries: object | object[] - regions: object | object[] - subdivisions: object | object[] - languages: object | object[] - categories: object | object[] - blocklist: object | object[] - channels: object | object[] - feeds: object | object[] - logos: object | object[] - timezones: object | object[] - guides: object | object[] - streams: object | object[] - cities: object | object[] -} diff --git a/scripts/types/dataProcessor.d.ts b/scripts/types/dataProcessor.d.ts deleted file mode 100644 index bc76dc28b4..0000000000 --- a/scripts/types/dataProcessor.d.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Collection, Dictionary } from '@freearhey/core' - -export type DataProcessorData = { - blocklistRecordsGroupedByChannelId: Dictionary - subdivisionsGroupedByCountryCode: Dictionary - feedsGroupedByChannelId: Dictionary - guidesGroupedByStreamId: Dictionary - logosGroupedByStreamId: Dictionary - subdivisionsKeyByCode: Dictionary - countriesKeyByCode: Dictionary - languagesKeyByCode: Dictionary - streamsGroupedById: Dictionary - categoriesKeyById: Dictionary - timezonesKeyById: Dictionary - regionsKeyByCode: Dictionary - blocklistRecords: Collection - channelsKeyById: Dictionary - citiesKeyByCode: Dictionary - subdivisions: Collection - categories: Collection - countries: Collection - languages: Collection - timezones: Collection - channels: Collection - regions: Collection - streams: Collection - cities: Collection - guides: Collection - feeds: Collection - logos: Collection -} diff --git a/scripts/types/feed.d.ts b/scripts/types/feed.d.ts deleted file mode 100644 index ef4aea4669..0000000000 --- a/scripts/types/feed.d.ts +++ /dev/null @@ -1,10 +0,0 @@ -export type FeedData = { - channel: string - id: string - name: string - is_main: boolean - broadcast_area: string[] - languages: string[] - timezones: string[] - video_format: string -} diff --git a/scripts/types/guide.d.ts b/scripts/types/guide.d.ts deleted file mode 100644 index 63a6ecdb19..0000000000 --- a/scripts/types/guide.d.ts +++ /dev/null @@ -1,17 +0,0 @@ -export type GuideSerializedData = { - channelId?: string - feedId?: string - siteDomain: string - siteId: string - siteName: string - languageCode: string -} - -export type GuideData = { - channel: string - feed: string - site: string - site_id: string - site_name: string - lang: string -} diff --git a/scripts/types/language.d.ts b/scripts/types/language.d.ts deleted file mode 100644 index 2b9d4525c0..0000000000 --- a/scripts/types/language.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -export type LanguageSerializedData = { - code: string - name: string -} - -export type LanguageData = { - code: string - name: string -} diff --git a/scripts/types/logo.d.ts b/scripts/types/logo.d.ts deleted file mode 100644 index 47cc384537..0000000000 --- a/scripts/types/logo.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -export type LogoData = { - channel: string - feed: string | null - tags: string[] - width: number - height: number - format: string | null - url: string -} diff --git a/scripts/types/region.d.ts b/scripts/types/region.d.ts deleted file mode 100644 index 798224ee7f..0000000000 --- a/scripts/types/region.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { CitySerializedData } from './city' -import { CountrySerializedData } from './country' -import { SubdivisionSerializedData } from './subdivision' - -export type RegionSerializedData = { - code: string - name: string - countryCodes: string[] - countries?: CountrySerializedData[] - subdivisions?: SubdivisionSerializedData[] - cities?: CitySerializedData[] -} - -export type RegionData = { - code: string - name: string - countries: string[] -} diff --git a/scripts/types/stream.d.ts b/scripts/types/stream.d.ts deleted file mode 100644 index aebd4bbd9f..0000000000 --- a/scripts/types/stream.d.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type StreamData = { - channelId: string | null - feedId: string | null - title: string | null - url: string - referrer: string | null - userAgent: string | null - quality: string | null - label: string | null - directives: string[] -} diff --git a/scripts/types/subdivision.d.ts b/scripts/types/subdivision.d.ts deleted file mode 100644 index b2a25982dd..0000000000 --- a/scripts/types/subdivision.d.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { CountrySerializedData } from './country' - -export type SubdivisionSerializedData = { - code: string - name: string - countryCode: string - country?: CountrySerializedData - parentCode: string | null -} - -export type SubdivisionData = { - code: string - name: string - country: string - parent: string | null -} diff --git a/scripts/utils.ts b/scripts/utils.ts index 7ff419de21..d58b80406d 100644 --- a/scripts/utils.ts +++ b/scripts/utils.ts @@ -1,8 +1,23 @@ -export function isURI(string: string): boolean { - try { - new URL(string) - return true - } catch { - return false - } -} +import normalizeUrl from 'normalize-url' + +export function isURI(string: string): boolean { + try { + new URL(string) + return true + } catch { + return false + } +} + +export function normalizeURL(url: string): string { + const normalized = normalizeUrl(url, { stripWWW: false }) + + return decodeURIComponent(normalized).replace(/\s/g, '+').toString() +} + +export function truncate(string: string, limit: number = 100) { + if (!string) return string + if (string.length < limit) return string + + return string.slice(0, limit - 3) + '...' +}