diff --git a/scripts/commands/issue/validate.ts b/scripts/commands/issue/validate.ts new file mode 100644 index 0000000000..c3e159a668 --- /dev/null +++ b/scripts/commands/issue/validate.ts @@ -0,0 +1,144 @@ +import { LOGS_DIR, STREAMS_DIR } from '../../constants' +import { loadData, data as apiData } from '../../api' +import { Collection, Logger } from '@freearhey/core' +import { hasValidDomain, isURI, parseIssueBody } from '../../utils' +import { Storage } from '@freearhey/storage-js' +import { PlaylistParser } from '../../core' +import { Stream } from '../../models' +import * as sdk from '@iptv-org/sdk' +import { program } from 'commander' + +program + .requiredOption('--body ', 'The full markdown body text of the issue') + .option('--labels ', 'Comma-separated string of label names', '') + .parse(process.argv) + +const { body, labels } = program.opts() + +const logsStorage = new Storage(LOGS_DIR) +let streams = new Collection() +const errors: string[] = [] + +async function main() { + const logger = new Logger() + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + await loadStreams() + + const data = parseIssueBody(body) + if (labels.includes('streams:add')) { + if (data.missing('stream_id')) { + errors.push('The request is missing the "Stream ID"') + done() + } + + const streamUrl = data.getString('stream_url') + if (!streamUrl) { + errors.push('The request is missing the "Stream URL"') + done() + } else if (!isURI(streamUrl) || !hasValidDomain(streamUrl)) { + errors.push(`The stream URL "${streamUrl}" is invalid`) + } + + if (streams.includes((_stream: Stream) => _stream.url === streamUrl)) { + errors.push(`The stream with the URL "${streamUrl}" is already included in the playlists`) + } + + const streamId = data.getString('stream_id') || '' + const [channelId, feedId] = streamId.split('@') + + const channel: sdk.Models.Channel | undefined = apiData.channelsKeyById.get(channelId) + if (!channel) { + errors.push(`There is no channel with the ID "${channelId}" in the database`) + done() + } + + const blocklistRecords: sdk.Models.BlocklistRecord[] | undefined = + apiData.blocklistRecordsGroupedByChannel.get(channelId) + if (blocklistRecords) { + blocklistRecords.forEach((record: sdk.Models.BlocklistRecord) => { + if (record.reason === 'dmca') { + errors.push( + `The channel "${channelId}" has been added to our blocklist due to the claims of the copyright holder: ${record.ref}` + ) + } else if (record.reason === 'nsfw') { + errors.push( + `The channel "${channelId}" has been added to our blocklist due to NSFW content: ${record.ref}` + ) + } + }) + } + + const feed: sdk.Models.Feed | undefined = apiData.feedsKeyByStreamId.get(streamId) + if (!feed) { + errors.push( + `There is no feed with the ID "${feedId}" for the "${channelId}" channel in the database` + ) + } + } else if (labels.includes('streams:remove')) { + const streamUrls = data.getString('stream_url') || '' + if (!streamUrls) { + errors.push('The request is missing the "Stream URL"') + done() + } + + streamUrls + .split(/\r?\n/) + .filter(Boolean) + .forEach((link: string) => { + if (!isURI(link) || !hasValidDomain(link)) { + errors.push(`The stream URL "${link}" is invalid`) + } + + const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim()) + if (!found) { + errors.push(`The stream with the URL "${link}" is missing from the playlists`) + } + }) + } else if (labels.includes('streams:edit')) { + const streamUrl = data.getString('stream_url') + + if (!streamUrl) { + errors.push('The request is missing the "Stream URL"') + done() + } else if (!isURI(streamUrl) || !hasValidDomain(streamUrl)) { + errors.push(`The stream URL "${streamUrl}" is invalid`) + done() + } + + const stream: Stream = streams.first((_stream: Stream) => _stream.url === streamUrl) + if (!stream) { + errors.push(`The stream with the URL "${streamUrl}" is missing from the playlists`) + } + } + + done() +} + +function done() { + if (errors.length) { + let message = 'The request contains error(s):' + errors.forEach(error => { + message += `\r\n- ${error}` + }) + logsStorage.saveSync('errors.txt', message) + process.exit(1) + } + + process.exit(0) +} + +async function loadStreams() { + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ + storage: streamsStorage + }) + const files = await streamsStorage.list('**/*.m3u') + + streams = await parser.parse(files) +} + +main() diff --git a/scripts/commands/playlist/update.ts b/scripts/commands/playlist/update.ts index f0477ed84e..7b0a6dbdad 100644 --- a/scripts/commands/playlist/update.ts +++ b/scripts/commands/playlist/update.ts @@ -109,7 +109,7 @@ async function removeStream(issue: Issue) { const log = createThread(issue, 'streams/remove') log.start() - const data = issue.data + const data = issue.dataSet if (data.missing('stream_url')) { log.error('The request is missing the "Stream URL"') skippedIssues.add(issue) @@ -145,7 +145,7 @@ async function editStream(issue: Issue) { const log = createThread(issue, 'streams/edit') log.start() - const data = issue.data + const data = issue.dataSet const streamUrl = data.getString('stream_url') @@ -157,7 +157,7 @@ async function editStream(issue: Issue) { const stream: Stream = streams.first((_stream: Stream) => _stream.url === streamUrl) if (!stream) { - log.error(`The stream with the URL "${streamUrl}" is already in the playlists`) + log.error(`The stream with the URL "${streamUrl}" is missing from the playlists`) skippedIssues.add(issue) return } @@ -189,7 +189,7 @@ async function addStream(issue: Issue) { const log = createThread(issue, 'streams/add') log.start() - const data = issue.data + const data = issue.dataSet if (data.missing('stream_id')) { log.error('The request is missing the "Stream ID"') skippedIssues.add(issue) diff --git a/scripts/commands/report/create.ts b/scripts/commands/report/create.ts index b1af2f4fb6..d6e94aef1a 100644 --- a/scripts/commands/report/create.ts +++ b/scripts/commands/report/create.ts @@ -49,7 +49,7 @@ async function main() { issue.labels.find((label: string) => label === 'streams:remove') ) removeRequests.forEach((issue: Issue) => { - const streamUrls = issue.data.getArray('stream_url') || [] + const streamUrls = issue.dataSet.getArray('stream_url') || [] if (!streamUrls.length) { const result = { @@ -84,8 +84,8 @@ async function main() { const addRequests = issues.filter(issue => issue.labels.includes('streams:add')) const addRequestsBuffer = new Dictionary() addRequests.forEach((issue: Issue) => { - const streamId = issue.data.getString('stream_id') || '' - const streamUrl = issue.data.getString('stream_url') || '' + const streamId = issue.dataSet.getString('stream_id') || '' + const streamUrl = issue.dataSet.getString('stream_url') || '' const [channelId] = streamId.split('@') const result = { @@ -116,8 +116,8 @@ async function main() { issue.labels.find((label: string) => label === 'streams:edit') ) editRequests.forEach((issue: Issue) => { - const streamId = issue.data.getString('stream_id') || '' - const streamUrl = issue.data.getString('stream_url') || '' + const streamId = issue.dataSet.getString('stream_id') || '' + const streamUrl = issue.dataSet.getString('stream_url') || '' const [channelId] = streamId.split('@') const result = { diff --git a/scripts/models/issue.ts b/scripts/models/issue.ts index 28458ce75d..ec18d63d1c 100644 --- a/scripts/models/issue.ts +++ b/scripts/models/issue.ts @@ -3,17 +3,17 @@ import { DataSet } from '../core' type IssueProps = { number: number labels: string[] - data: DataSet + dataSet: DataSet } export class Issue { number: number labels: string[] - data: DataSet + dataSet: DataSet - constructor({ number, labels, data }: IssueProps) { + constructor({ number, labels, dataSet }: IssueProps) { this.number = number this.labels = labels - this.data = data + this.dataSet = dataSet } } diff --git a/scripts/utils.ts b/scripts/utils.ts index 8909e9dba9..2b354e6657 100644 --- a/scripts/utils.ts +++ b/scripts/utils.ts @@ -15,6 +15,20 @@ import { orderBy } from 'es-toolkit' import path from 'node:path' import fs from 'node:fs' +export function hasValidDomain(string: string): boolean { + const FORBIDDEN_DOMAINS = ['iptv-org.github.io'] + try { + const parsedUrl = new URL(string) + const hostname = parsedUrl.hostname + const isForbidden = FORBIDDEN_DOMAINS.some( + domain => hostname === domain || hostname.endsWith(`.${domain}`) + ) + return !isForbidden + } catch { + return false + } +} + export function isURI(string: string): boolean { try { const url = new URL(string) @@ -174,10 +188,13 @@ export async function loadIssues(props?: { labels: string | string[] }) { }) } - return new Collection(issues).map(parseIssue) + return new Collection(issues).map(({ number, body, labels }) => { + const dataSet = parseIssueBody(body) + return new Issue({ number, labels: labels.map(l => l.name), dataSet }) + }) } -function parseIssue(issue: { number: number; body: string; labels: { name: string }[] }): Issue { +export function parseIssueBody(body: string): DataSet { const FIELDS = new Dictionary({ 'Stream ID': 'stream_id', 'Channel ID': 'channel_id', @@ -193,7 +210,7 @@ function parseIssue(issue: { number: number; body: string; labels: { name: strin Notes: 'notes' }) - const fields = typeof issue.body === 'string' ? issue.body.split('###') : [] + const fields = typeof body === 'string' ? body.split('###') : [] const data = new Dictionary() fields.forEach((field: string) => { @@ -213,9 +230,7 @@ function parseIssue(issue: { number: number; body: string; labels: { name: strin data.set(id, value) }) - const labels = issue.labels.map(label => label.name) - - return new Issue({ number: issue.number, labels, data: new DataSet(data) }) + return new DataSet(data) } export async function loadDiscussions() {