diff --git a/.github/docs/scripts.md b/.github/docs/scripts.md index 95068c86a7..79edeebe72 100644 --- a/.github/docs/scripts.md +++ b/.github/docs/scripts.md @@ -7,6 +7,8 @@ For the scripts to work, you must have [Node.js](https://nodejs.org/en) installe - [act:check](#actcheck) - [act:format](#actformat) - [act:update](#actupdate) +- [act:validate_issue](#actvalidate_issue) +- [act:validate_label](#actvalidate_label) - [api:load](#apiload) - [playlist:format](#playlistformat) - [playlist:update](#playlistupdate) @@ -45,6 +47,22 @@ Runs the [update](./workflows.md#update) workflow locally. Depends on [nektos/gh npm run act:update ``` +## act:validate_issue + +Runs the [validate_issue](./workflows.md#validate_issue) workflow locally. Depends on [nektos/gh-act](https://github.com/nektos/gh-act). + +```sh +npm run act:validate_issue -- -e .github/mocks/events/issue_opened.json +``` + +## act:validate_label + +Runs the [validate_label](./workflows.md#validate_label) workflow locally. Depends on [nektos/gh-act](https://github.com/nektos/gh-act). + +```sh +npm run act:validate_label -- -e .github/mocks/events/issue_approved.json --actor Bob +``` + ## api:load Downloads the latest channel and stream data from the [iptv-org/api](https://github.com/iptv-org/api) repository. diff --git a/.github/docs/workflows.md b/.github/docs/workflows.md index 4ca3d326d7..d70b724dea 100644 --- a/.github/docs/workflows.md +++ b/.github/docs/workflows.md @@ -15,3 +15,11 @@ Sequentially runs the `api:load`, `playlist:format`, `playlist:lint`, and `playl ## update Runs every day at 0:00 UTC. It sequentially executes the `api:load`, `playlist:update`, `playlist:lint`, `playlist:validate`, `playlist:generate`, `playlist:export`, and `readme:update` scripts, then automatically deploys the updated files if successful. + +## validate_issue + +Runs when an issue is opened or edited. It checks the requests for errors using `issue:validate` script and, if any are found, the bot posts them in the comments. + +## validate_label + +Triggers when a new label is added to an issue. It checks the label, and if it's incorrect, the bot deletes it and leaves a comment explaining why. diff --git a/.github/mocks/events/issue_approved.json b/.github/mocks/events/issue_approved.json new file mode 100644 index 0000000000..9d4fb1ea33 --- /dev/null +++ b/.github/mocks/events/issue_approved.json @@ -0,0 +1,12 @@ +{ + "action": "labeled", + "label": { + "name": "approved" + }, + "issue": { + "number": 48372, + "user": { + "login": "Bob" + } + } +} \ No newline at end of file diff --git a/.github/mocks/events/issue_edited.json b/.github/mocks/events/issue_edited.json new file mode 100644 index 0000000000..a581579de2 --- /dev/null +++ b/.github/mocks/events/issue_edited.json @@ -0,0 +1,14 @@ +{ + "action": "edited", + "issue": { + "number": 48372, + "title": "[BUG] Title", + "body": "Lorem ipsum", + "user": { + "login": "Bob" + }, + "labels": [ + { "name": "bug" } + ] + } +} \ No newline at end of file diff --git a/.github/mocks/events/issue_opened.json b/.github/mocks/events/issue_opened.json new file mode 100644 index 0000000000..0c8c61eafa --- /dev/null +++ b/.github/mocks/events/issue_opened.json @@ -0,0 +1,14 @@ +{ + "action": "opened", + "issue": { + "number": 48372, + "title": "Add: WLTV-DT 23.1 HD", + "body": "### Stream ID (required)\n\nWLTVDT231.us@East\n\n### Stream URL (required)\n\nhttps://iptv-org.github.io/iptv/index.m3u\n\n### Quality\n\n720p\n\n### Label\n\n_No response_\n\n### HTTP User Agent\n\n_No response_\n\n### HTTP Referrer\n\n_No response_\n\n### Notes\n\n_No response_\n\n### Contributing Guide\n\n- [x] I have read [Contributing Guide](https://github.com/iptv-org/iptv/blob/master/CONTRIBUTING.md)", + "user": { + "login": "Bob" + }, + "labels": [ + { "name": "streams:add" } + ] + } +} \ No newline at end of file diff --git a/.github/workflows/validate_issue.yml b/.github/workflows/validate_issue.yml new file mode 100644 index 0000000000..908d57eb34 --- /dev/null +++ b/.github/workflows/validate_issue.yml @@ -0,0 +1,75 @@ +name: Validate Issue +on: + issues: + types: [opened, edited] + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + if: | + contains(github.event.issue.labels.*.name, 'streams:add') || + contains(github.event.issue.labels.*.name, 'streams:remove') || + contains(github.event.issue.labels.*.name, 'streams:edit') + steps: + - name: Checkout repository code + uses: actions/checkout@v6 + + - uses: tibdex/github-app-token@v1.8.2 + if: ${{ !env.ACT }} + id: create-app-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run issue:validate + id: validator + env: + RAW_BODY: ${{ github.event.issue.body }} + LABEL_STRING: ${{ join(github.event.issue.labels.*.name, ',') }} + run: | + npm run issue:validate -- \ + --body "$RAW_BODY" \ + --labels "$LABEL_STRING" + + - name: Handle valid request + if: success() + env: + GH_TOKEN: ${{ steps.create-app-token.outputs.token }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + run: | + if [ "${{ env.ACT }}" = "true" ]; then + echo "[MOCK] Add label: check:passed" + echo "[MOCK] Remove label if exists: check:failed" + else + gh issue edit $ISSUE_NUMBER --add-label "check:passed" --remove-label "check:failed" + fi + + - name: Handle invalid request + if: failure() && steps.validator.outcome == 'failure' + env: + GH_TOKEN: ${{ steps.create-app-token.outputs.token }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + run: | + COMMENT_MSG=$(cat temp/logs/errors.txt) + + if [ "${{ env.ACT }}" = "true" ]; then + echo "[MOCK] Post comment:" + echo "$COMMENT_MSG" + echo "[MOCK] Add label: check:failed" + echo "[MOCK] Remove label if exists: check:passed" + else + gh issue comment $ISSUE_NUMBER --body "$COMMENT_MSG" + gh issue edit $ISSUE_NUMBER --add-label "check:failed" --remove-label "check:passed" + fi diff --git a/.github/workflows/validate_label.yml b/.github/workflows/validate_label.yml new file mode 100644 index 0000000000..7d6096930c --- /dev/null +++ b/.github/workflows/validate_label.yml @@ -0,0 +1,40 @@ +name: Validate Label +on: + issues: + types: [labeled] + +permissions: + contents: read + issues: write + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: Checkout repository code + uses: actions/checkout@v6 + + - uses: tibdex/github-app-token@v1.8.2 + if: ${{ !env.ACT }} + id: create-app-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Block self-approval + if: | + github.event.label.name == 'approved' && + github.actor == github.event.issue.user.login + env: + GH_TOKEN: ${{ steps.create-app-token.outputs.token }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + run: | + if [ "${{ env.ACT }}" = "true" ]; then + echo "[MOCK] Post comment: A request cannot be approved by its author." + echo "[MOCK] Remove label: approved" + else + gh issue comment $ISSUE_NUMBER --body "A request cannot be approved by its author." + gh issue edit $ISSUE_NUMBER --remove-label "approved" + fi + + exit 1 diff --git a/package.json b/package.json index a0d8fdfed1..ee69a9d98a 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,10 @@ "act:check": "gh act pull_request -W .github/workflows/check.yml", "act:format": "gh act workflow_dispatch -W .github/workflows/format.yml", "act:update": "gh act workflow_dispatch -W .github/workflows/update.yml", + "act:validate_issue": "gh act issues -W .github/workflows/validate_issue.yml", + "act:validate_label": "gh act issues -W .github/workflows/validate_label.yml", "api:load": "tsx scripts/commands/api/load.ts", + "issue:validate": "tsx scripts/commands/issue/validate.ts", "playlist:format": "tsx scripts/commands/playlist/format.ts", "playlist:update": "tsx scripts/commands/playlist/update.ts", "playlist:generate": "tsx scripts/commands/playlist/generate.ts", diff --git a/scripts/commands/issue/validate.ts b/scripts/commands/issue/validate.ts new file mode 100644 index 0000000000..c3e159a668 --- /dev/null +++ b/scripts/commands/issue/validate.ts @@ -0,0 +1,144 @@ +import { LOGS_DIR, STREAMS_DIR } from '../../constants' +import { loadData, data as apiData } from '../../api' +import { Collection, Logger } from '@freearhey/core' +import { hasValidDomain, isURI, parseIssueBody } from '../../utils' +import { Storage } from '@freearhey/storage-js' +import { PlaylistParser } from '../../core' +import { Stream } from '../../models' +import * as sdk from '@iptv-org/sdk' +import { program } from 'commander' + +program + .requiredOption('--body ', 'The full markdown body text of the issue') + .option('--labels ', 'Comma-separated string of label names', '') + .parse(process.argv) + +const { body, labels } = program.opts() + +const logsStorage = new Storage(LOGS_DIR) +let streams = new Collection() +const errors: string[] = [] + +async function main() { + const logger = new Logger() + + logger.info('loading data from api...') + await loadData() + + logger.info('loading streams...') + await loadStreams() + + const data = parseIssueBody(body) + if (labels.includes('streams:add')) { + if (data.missing('stream_id')) { + errors.push('The request is missing the "Stream ID"') + done() + } + + const streamUrl = data.getString('stream_url') + if (!streamUrl) { + errors.push('The request is missing the "Stream URL"') + done() + } else if (!isURI(streamUrl) || !hasValidDomain(streamUrl)) { + errors.push(`The stream URL "${streamUrl}" is invalid`) + } + + if (streams.includes((_stream: Stream) => _stream.url === streamUrl)) { + errors.push(`The stream with the URL "${streamUrl}" is already included in the playlists`) + } + + const streamId = data.getString('stream_id') || '' + const [channelId, feedId] = streamId.split('@') + + const channel: sdk.Models.Channel | undefined = apiData.channelsKeyById.get(channelId) + if (!channel) { + errors.push(`There is no channel with the ID "${channelId}" in the database`) + done() + } + + const blocklistRecords: sdk.Models.BlocklistRecord[] | undefined = + apiData.blocklistRecordsGroupedByChannel.get(channelId) + if (blocklistRecords) { + blocklistRecords.forEach((record: sdk.Models.BlocklistRecord) => { + if (record.reason === 'dmca') { + errors.push( + `The channel "${channelId}" has been added to our blocklist due to the claims of the copyright holder: ${record.ref}` + ) + } else if (record.reason === 'nsfw') { + errors.push( + `The channel "${channelId}" has been added to our blocklist due to NSFW content: ${record.ref}` + ) + } + }) + } + + const feed: sdk.Models.Feed | undefined = apiData.feedsKeyByStreamId.get(streamId) + if (!feed) { + errors.push( + `There is no feed with the ID "${feedId}" for the "${channelId}" channel in the database` + ) + } + } else if (labels.includes('streams:remove')) { + const streamUrls = data.getString('stream_url') || '' + if (!streamUrls) { + errors.push('The request is missing the "Stream URL"') + done() + } + + streamUrls + .split(/\r?\n/) + .filter(Boolean) + .forEach((link: string) => { + if (!isURI(link) || !hasValidDomain(link)) { + errors.push(`The stream URL "${link}" is invalid`) + } + + const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim()) + if (!found) { + errors.push(`The stream with the URL "${link}" is missing from the playlists`) + } + }) + } else if (labels.includes('streams:edit')) { + const streamUrl = data.getString('stream_url') + + if (!streamUrl) { + errors.push('The request is missing the "Stream URL"') + done() + } else if (!isURI(streamUrl) || !hasValidDomain(streamUrl)) { + errors.push(`The stream URL "${streamUrl}" is invalid`) + done() + } + + const stream: Stream = streams.first((_stream: Stream) => _stream.url === streamUrl) + if (!stream) { + errors.push(`The stream with the URL "${streamUrl}" is missing from the playlists`) + } + } + + done() +} + +function done() { + if (errors.length) { + let message = 'The request contains error(s):' + errors.forEach(error => { + message += `\r\n- ${error}` + }) + logsStorage.saveSync('errors.txt', message) + process.exit(1) + } + + process.exit(0) +} + +async function loadStreams() { + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ + storage: streamsStorage + }) + const files = await streamsStorage.list('**/*.m3u') + + streams = await parser.parse(files) +} + +main() diff --git a/scripts/commands/playlist/update.ts b/scripts/commands/playlist/update.ts index f0477ed84e..7b0a6dbdad 100644 --- a/scripts/commands/playlist/update.ts +++ b/scripts/commands/playlist/update.ts @@ -109,7 +109,7 @@ async function removeStream(issue: Issue) { const log = createThread(issue, 'streams/remove') log.start() - const data = issue.data + const data = issue.dataSet if (data.missing('stream_url')) { log.error('The request is missing the "Stream URL"') skippedIssues.add(issue) @@ -145,7 +145,7 @@ async function editStream(issue: Issue) { const log = createThread(issue, 'streams/edit') log.start() - const data = issue.data + const data = issue.dataSet const streamUrl = data.getString('stream_url') @@ -157,7 +157,7 @@ async function editStream(issue: Issue) { const stream: Stream = streams.first((_stream: Stream) => _stream.url === streamUrl) if (!stream) { - log.error(`The stream with the URL "${streamUrl}" is already in the playlists`) + log.error(`The stream with the URL "${streamUrl}" is missing from the playlists`) skippedIssues.add(issue) return } @@ -189,7 +189,7 @@ async function addStream(issue: Issue) { const log = createThread(issue, 'streams/add') log.start() - const data = issue.data + const data = issue.dataSet if (data.missing('stream_id')) { log.error('The request is missing the "Stream ID"') skippedIssues.add(issue) diff --git a/scripts/commands/report/create.ts b/scripts/commands/report/create.ts index b1af2f4fb6..d6e94aef1a 100644 --- a/scripts/commands/report/create.ts +++ b/scripts/commands/report/create.ts @@ -49,7 +49,7 @@ async function main() { issue.labels.find((label: string) => label === 'streams:remove') ) removeRequests.forEach((issue: Issue) => { - const streamUrls = issue.data.getArray('stream_url') || [] + const streamUrls = issue.dataSet.getArray('stream_url') || [] if (!streamUrls.length) { const result = { @@ -84,8 +84,8 @@ async function main() { const addRequests = issues.filter(issue => issue.labels.includes('streams:add')) const addRequestsBuffer = new Dictionary() addRequests.forEach((issue: Issue) => { - const streamId = issue.data.getString('stream_id') || '' - const streamUrl = issue.data.getString('stream_url') || '' + const streamId = issue.dataSet.getString('stream_id') || '' + const streamUrl = issue.dataSet.getString('stream_url') || '' const [channelId] = streamId.split('@') const result = { @@ -116,8 +116,8 @@ async function main() { issue.labels.find((label: string) => label === 'streams:edit') ) editRequests.forEach((issue: Issue) => { - const streamId = issue.data.getString('stream_id') || '' - const streamUrl = issue.data.getString('stream_url') || '' + const streamId = issue.dataSet.getString('stream_id') || '' + const streamUrl = issue.dataSet.getString('stream_url') || '' const [channelId] = streamId.split('@') const result = { diff --git a/scripts/models/issue.ts b/scripts/models/issue.ts index 28458ce75d..ec18d63d1c 100644 --- a/scripts/models/issue.ts +++ b/scripts/models/issue.ts @@ -3,17 +3,17 @@ import { DataSet } from '../core' type IssueProps = { number: number labels: string[] - data: DataSet + dataSet: DataSet } export class Issue { number: number labels: string[] - data: DataSet + dataSet: DataSet - constructor({ number, labels, data }: IssueProps) { + constructor({ number, labels, dataSet }: IssueProps) { this.number = number this.labels = labels - this.data = data + this.dataSet = dataSet } } diff --git a/scripts/utils.ts b/scripts/utils.ts index 8909e9dba9..2b354e6657 100644 --- a/scripts/utils.ts +++ b/scripts/utils.ts @@ -15,6 +15,20 @@ import { orderBy } from 'es-toolkit' import path from 'node:path' import fs from 'node:fs' +export function hasValidDomain(string: string): boolean { + const FORBIDDEN_DOMAINS = ['iptv-org.github.io'] + try { + const parsedUrl = new URL(string) + const hostname = parsedUrl.hostname + const isForbidden = FORBIDDEN_DOMAINS.some( + domain => hostname === domain || hostname.endsWith(`.${domain}`) + ) + return !isForbidden + } catch { + return false + } +} + export function isURI(string: string): boolean { try { const url = new URL(string) @@ -174,10 +188,13 @@ export async function loadIssues(props?: { labels: string | string[] }) { }) } - return new Collection(issues).map(parseIssue) + return new Collection(issues).map(({ number, body, labels }) => { + const dataSet = parseIssueBody(body) + return new Issue({ number, labels: labels.map(l => l.name), dataSet }) + }) } -function parseIssue(issue: { number: number; body: string; labels: { name: string }[] }): Issue { +export function parseIssueBody(body: string): DataSet { const FIELDS = new Dictionary({ 'Stream ID': 'stream_id', 'Channel ID': 'channel_id', @@ -193,7 +210,7 @@ function parseIssue(issue: { number: number; body: string; labels: { name: strin Notes: 'notes' }) - const fields = typeof issue.body === 'string' ? issue.body.split('###') : [] + const fields = typeof body === 'string' ? body.split('###') : [] const data = new Dictionary() fields.forEach((field: string) => { @@ -213,9 +230,7 @@ function parseIssue(issue: { number: number; body: string; labels: { name: strin data.set(id, value) }) - const labels = issue.labels.map(label => label.name) - - return new Issue({ number: issue.number, labels, data: new DataSet(data) }) + return new DataSet(data) } export async function loadDiscussions() { diff --git a/tests/__data__/expected/issue_validate/logs/streams_add.txt b/tests/__data__/expected/issue_validate/logs/streams_add.txt new file mode 100644 index 0000000000..92f3af9505 --- /dev/null +++ b/tests/__data__/expected/issue_validate/logs/streams_add.txt @@ -0,0 +1,2 @@ +The request contains error(s): +- There is no feed with the ID "undefined" for the "ManoramaNews.in" channel in the database \ No newline at end of file diff --git a/tests/__data__/expected/issue_validate/logs/streams_edit.txt b/tests/__data__/expected/issue_validate/logs/streams_edit.txt new file mode 100644 index 0000000000..5643d628b3 --- /dev/null +++ b/tests/__data__/expected/issue_validate/logs/streams_edit.txt @@ -0,0 +1,2 @@ +The request contains error(s): +- The stream with the URL "https://livestream.telvue.com/templeuni1/f7b44cfafd5c52223d5498196c8a2e7b.sdp/playlist.m3u8" is missing from the playlists \ No newline at end of file diff --git a/tests/__data__/expected/issue_validate/logs/streams_remove.txt b/tests/__data__/expected/issue_validate/logs/streams_remove.txt new file mode 100644 index 0000000000..d7ba217ca5 --- /dev/null +++ b/tests/__data__/expected/issue_validate/logs/streams_remove.txt @@ -0,0 +1,3 @@ +The request contains error(s): +- The stream with the URL "http://l6.cloudskep.com/rikcy/rikhd/playlist.m3u8" is missing from the playlists +- The stream with the URL "http://l6.cloudskep.com/rikcy/rik2/playlist.m3u8" is missing from the playlists \ No newline at end of file diff --git a/tests/__data__/input/issue_validate/streams/br.m3u b/tests/__data__/input/issue_validate/streams/br.m3u new file mode 100644 index 0000000000..6a78c15f31 --- /dev/null +++ b/tests/__data__/input/issue_validate/streams/br.m3u @@ -0,0 +1,3 @@ +#EXTM3U +#EXTINF:-1 tvg-id="",VTV [Not 24/7] +https://ythls.onrender.com/channel/UC40TUSUx490U5uR1lZt3Ajg.m3u8 diff --git a/tests/commands/issue/validate.test.ts b/tests/commands/issue/validate.test.ts new file mode 100644 index 0000000000..061fb65dc7 --- /dev/null +++ b/tests/commands/issue/validate.test.ts @@ -0,0 +1,63 @@ +import issues from '../../__data__/input/issues' +import { pathToFileURL } from 'node:url' +import { execSync } from 'child_process' +import * as fs from 'fs-extra' + +const ENV_VAR = + 'cross-env DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/output/streams LOGS_DIR=tests/__data__/output/logs' + +beforeEach(() => { + fs.emptyDirSync('tests/__data__/output') + fs.copySync('tests/__data__/input/issue_validate/streams', 'tests/__data__/output/streams') +}) + +describe('issue:validate', () => { + it('can handle streams:add request', () => { + const body = issues.find(issue => issue.number === 14179)?.body + const cmd = `${ENV_VAR} npm run issue:validate --- --body="${body}" --labels="approved,streams:add"` + + try { + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + throw new Error('Failed') + } catch { + expect(content('tests/__data__/output/logs/errors.txt')).toBe( + content('tests/__data__/expected/issue_validate/logs/streams_add.txt') + ) + } + }) + + it('can handle streams:remove request', () => { + const body = issues.find(issue => issue.number === 14150)?.body + const cmd = `${ENV_VAR} npm run issue:validate --- --body="${body}" --labels="approved,streams:remove"` + + try { + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + throw new Error('Failed') + } catch { + expect(content('tests/__data__/output/logs/errors.txt')).toBe( + content('tests/__data__/expected/issue_validate/logs/streams_remove.txt') + ) + } + }) + + it('can handle streams:edit request', () => { + const body = issues.find(issue => issue.number === 14120)?.body + const cmd = `${ENV_VAR} npm run issue:validate --- --body="${body}" --labels="approved,streams:edit"` + + try { + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + throw new Error('Failed') + } catch { + expect(content('tests/__data__/output/logs/errors.txt')).toBe( + content('tests/__data__/expected/issue_validate/logs/streams_edit.txt') + ) + } + }) +}) + +function content(filepath: string) { + return fs.readFileSync(pathToFileURL(filepath), { encoding: 'utf8' }) +}