From d86a6ae494ead0e32a6a156bb97824c1061b471a Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 7 Nov 2022 21:12:12 +0300 Subject: [PATCH 1/6] Create __data__/ --- sites/foxtel.com.au/__data__/content.html | 4874 ++++++++++++++++++ sites/foxtel.com.au/__data__/no-content.html | 3810 ++++++++++++++ 2 files changed, 8684 insertions(+) create mode 100644 sites/foxtel.com.au/__data__/content.html create mode 100644 sites/foxtel.com.au/__data__/no-content.html diff --git a/sites/foxtel.com.au/__data__/content.html b/sites/foxtel.com.au/__data__/content.html new file mode 100644 index 00000000..88e604ef --- /dev/null +++ b/sites/foxtel.com.au/__data__/content.html @@ -0,0 +1,4874 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + TV Guide + + + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + +
+
+
+ + + + + +
+ +
+
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+ +
+ + +
+ + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + +
+ +
+
+
+
+
+ +

TV GUIDE

+ +
+
+ + + +
+ + + +
+ +
    +
+
+
+ + + +
+
+
+
+ +
+
+ + + + + + + + + +
+ +
+ +
+
+
+ + FULL GUIDE + +
+
+ +
+ + 100 + + Channel 9 Sydney + + +
+ + + +
+
+
+ +
+
+
+
+
+ +
+
+
+ + + + +
+ + 11:40pm Monday + +
+
+ +
+
+ The Equalizer + +
+ + + S1 + , + + + + Ep2 + , + + + Glory + + (M) + +
+
+
+
+
+
+ + The Equalizer Image + +
+
+
+
+ + + + +
+ + 12:30am + +
+
+ +
+
+ Emergence + +
+ + + S1 + , + + + + Ep13 + , + + + Killshot (2) + + (M) + +
+
+
+
+
+
+ + Emergence Image + +
+
+
+
+ + + + +
+ + 1:30am + +
+
+ +
+
+ TV Shop - Home Shopping + +
+ + Perfect Fit Bed + + (G) + +
+
+
+
+
+
+ + TV Shop - Home Shopping Image + +
+
+
+
+ + + + +
+ + 2:00am + +
+
+ +
+
+ TV Shop - Home Shopping + +
+ + Total Gym FIT + + (G) + +
+
+
+
+
+
+ + TV Shop - Home Shopping Image + +
+
+
+
+ + + + +
+ + 2:30am + +
+
+ +
+
+ Global Shop + +
+ + Copper Chef Titan Pan + + (G) + +
+
+
+
+
+
+ + Global Shop Image + +
+
+
+
+ + + + +
+ + 3:00am + +
+
+ +
+
+ TV Shop - Home Shopping + +
+ + TEBO Massage Chair + + (G) + +
+
+
+
+
+
+ + TV Shop - Home Shopping Image + +
+
+
+
+ + + + +
+ + 3:30am + +
+
+ +
+
+ TV Shop - Home Shopping + +
+ + Transforma Ladders + + (G) + +
+
+
+
+
+
+ + TV Shop - Home Shopping Image + +
+
+
+
+ + + + +
+ + 4:00am + +
+
+ +
+
+ Believer's Voice of Victory + +
+ + God's WORD Is Filled With Power + + (PG) + +
+
+
+
+
+
+ + Believer's Voice of Victory Image + +
+
+
+
+ + + + +
+ + 4:30am + +
+
+ +
+
+ A Current Affair + +
+ + + +
+
+
+
+
+
+ + A Current Affair Image + +
+
+
+
+ + + + +
+ + 5:00am + +
+
+ +
+
+ Nine News Early Edition + +
+ + + +
+
+
+
+
+
+ + Nine News Early Edition Image + +
+
+
+
+ + + + +
+ + 5:30am + +
+
+ +
+
+ Today + +
+ + + S2022 + , + + + + Ep222 + + + + + +
+
+
+
+
+
+ + Today Image + +
+
+
+
+ + + + +
+ + 9:00am + +
+
+ +
+
+ Today Extra + +
+ + + + (PG) + +
+
+
+
+
+
+ + Today Extra Image + +
+
+
+
+ + + + +
+ + 11:30am + +
+
+ +
+
+ Nine Morning News + +
+ + + +
+
+
+
+
+
+ + Nine Morning News Image + +
+
+
+
+ + + + +
+ + 12:00pm + +
+
+ +
+
+ My Mum, Your Dad + +
+ + + S1 + , + + + + Ep1 + , + + + We're Like The FBI Right Now + + (M) + +
+
+
+
+
+
+ + My Mum, Your Dad Image + +
+
+
+
+ + + + +
+ + 1:50pm + +
+
+ +
+
+ Talking Honey + +
+ + + S2022 + , + + + + Ep3 + , + + + Cultural Tokenism or Awareness + + (PG) + +
+
+
+
+
+
+ + Talking Honey Image + +
+
+
+
+ + + + +
+ + 2:00pm + +
+
+ +
+
+ Pointless + +
+ + + S27 + , + + + + Ep35 + + + + + + (PG) + +
+
+
+
+
+
+ + Pointless Image + +
+
+
+
+ + + + +
+ + 3:00pm + +
+
+ +
+
+ Tipping Point + +
+ + + S11 + , + + + + Ep154 + + + + + + (PG) + +
+
+
+
+
+
+ + Tipping Point Image + +
+
+
+
+ + + + +
+ + 4:00pm + +
+
+ +
+
+ Nine Afternoon News + +
+ + + +
+
+
+
+
+
+ + Nine Afternoon News Image + +
+
+
+
+ + + + +
+ + 5:00pm + +
+
+ +
+
+ Millionaire Hot Seat + +
+ + + S2022 + , + + + + Ep2466 + + + + + + (G) + +
+
+
+
+
+
+ + Millionaire Hot Seat Image + +
+
+
+
+ + + + +
+ + 6:00pm + +
+
+ +
+
+ Nine News + +
+ + + +
+
+
+
+
+
+ + Nine News Image + +
+
+
+
+ + + + +
+ + 7:00pm + +
+
+ +
+
+ A Current Affair + +
+ + + +
+
+
+
+
+
+ + A Current Affair Image + +
+
+
+
+ + + + +
+ + 7:30pm + +
+
+ +
+
+ My Mum, Your Dad + +
+ + + S1 + , + + + + Ep2 + , + + + This Might Get a Little Weird + + (M) + +
+
+
+
+
+
+ + My Mum, Your Dad Image + +
+
+
+
+ + + + +
+ + 9:10pm + +
+
+ +
+
+ Travel Guides + +
+ + + S4 + , + + + + Ep2 + , + + + Red Centre + + (PG) + +
+
+
+
+
+
+ + Travel Guides Image + +
+
+
+
+ + + + +
+ + 10:10pm + +
+
+ +
+
+ Botched + +
+ + + S6 + , + + + + Ep16 + , + + + Cross-Eyed Nips and Cartel Hips + + (M) + +
+
+
+
+
+
+ + Botched Image + +
+
+
+
+ + + + +
+ + 11:10pm + +
+
+ +
+
+ Nine News Late + +
+ + + +
+
+
+
+
+
+ +
+
+
+
+ + + + +
+ + 11:40pm + +
+
+ +
+
+ Skin + +
+ + + S1 + , + + + + Ep5 + + + + + + (M) + +
+
+
+
+
+
+ + Skin Image + +
+
+
+
+ + + +
+
+
+
+ + + + + + + + + + + + +
+ +
+
+
+
+
+
+
+ +
+
+
+
+
+
+

Bundles+

+ +
+
+

Services+

+ +
+
+

Support+

+ +
+
+

Account+

+ +
+
+

Trending+

+ +
+ +
+
+
+ + +
+
+ + + +
+
+
+
+
+
+
+
+
+
+
+ Foxtel Group Walking Together RAP artwork +
+
+
+
Acknowledgement of Country
+
Foxtel acknowledges the Traditional Owners and Custodians of Country throughout Australia. We pay our respects to all First Nations people and acknowledge Elders past and present.

Read more about Foxtel's commitment to reconciliation
+
+
+
+
+
 
+
+
+
+ + +
+
+
+
+
+
+ Foxtel Group Walking Together RAP artwork +
+
+
+
Acknowledgement of Country
+
Foxtel acknowledges the Traditional Owners and Custodians of Country throughout Australia. We pay our respects to all First Nations people and acknowledge Elders past and present.

Read more about Foxtel's commitment to reconciliation
+
+
+
+
+
 
+
+
+
+
+ + +
+
+
+
+
+
+
+
Foxtel Group Walking Together RAP artwork +
+
+
Acknowledgement of Country
+
+
+
+
+

Foxtel acknowledges the Traditional Owners and Custodians of Country throughout Australia. We pay our respects to all First Nations people and acknowledge Elders past and present.

+ Read more about Foxtel's commitment to reconciliation +
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+ +
+
+
+ +

© 2022 Foxtel

+ +
+ +
+
+
+
+ + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+ + + diff --git a/sites/foxtel.com.au/__data__/no-content.html b/sites/foxtel.com.au/__data__/no-content.html new file mode 100644 index 00000000..e5890649 --- /dev/null +++ b/sites/foxtel.com.au/__data__/no-content.html @@ -0,0 +1,3810 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + TV Guide + + + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + +
+
+
+ + + + + +
+ +
+
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+ +
+ + +
+ + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + +
+ +
+
+
+
+
+ +

TV GUIDE

+ +
+
+ + + +
+ + + +
+ +
    +
+
+
+ + + +
+
+
+
+ +
+
+ + + + + + + + + +
+ +
+ +
+
+
+ + FULL GUIDE + +
+
+ +
+ + 100 + + Channel 9 Sydney + + +
+ + + +
+
+
+ +
+
+
+
+
+ +
+
+
+ + + +
+
+
+
+ + + + + + + + + + + + +
+ +
+
+
+
+
+
+
+ +
+
+
+
+
+
+

Bundles+

+ +
+
+

Services+

+ +
+
+

Support+

+ +
+
+

Account+

+ +
+
+

Trending+

+ +
+ +
+
+
+ + +
+
+ + + +
+
+
+
+
+
+
+
+
+
+
+ Foxtel Group Walking Together RAP artwork +
+
+
+
Acknowledgement of Country
+
Foxtel acknowledges the Traditional Owners and Custodians of Country throughout Australia. We pay our respects to all First Nations people and acknowledge Elders past and present.

Read more about Foxtel's commitment to reconciliation
+
+
+
+
+
 
+
+
+
+ + +
+
+
+
+
+
+ Foxtel Group Walking Together RAP artwork +
+
+
+
Acknowledgement of Country
+
Foxtel acknowledges the Traditional Owners and Custodians of Country throughout Australia. We pay our respects to all First Nations people and acknowledge Elders past and present.

Read more about Foxtel's commitment to reconciliation
+
+
+
+
+
 
+
+
+
+
+ + +
+
+
+
+
+
+
+
Foxtel Group Walking Together RAP artwork +
+
+
Acknowledgement of Country
+
+
+
+
+

Foxtel acknowledges the Traditional Owners and Custodians of Country throughout Australia. We pay our respects to all First Nations people and acknowledge Elders past and present.

+ Read more about Foxtel's commitment to reconciliation +
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+ +
+
+
+ +

© 2022 Foxtel

+ +
+ +
+
+
+
+ + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+ + + From 3ff1815e5d8a61dc78ed5eadf0047f2837343bca Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 7 Nov 2022 21:12:16 +0300 Subject: [PATCH 2/6] Create foxtel.com.au.test.js --- sites/foxtel.com.au/foxtel.com.au.test.js | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 sites/foxtel.com.au/foxtel.com.au.test.js diff --git a/sites/foxtel.com.au/foxtel.com.au.test.js b/sites/foxtel.com.au/foxtel.com.au.test.js new file mode 100644 index 00000000..9c96562c --- /dev/null +++ b/sites/foxtel.com.au/foxtel.com.au.test.js @@ -0,0 +1,62 @@ +// npm run channels:parse -- --config=./sites/foxtel.com.au/foxtel.com.au.config.js --output=./sites/foxtel.com.au/foxtel.com.au_au.channels.xml +// npx epg-grabber --config=sites/foxtel.com.au/foxtel.com.au.config.js --channels=sites/foxtel.com.au/foxtel.com.au_au.channels.xml --output=guide.xml --days=2 + +const { parser, url, request } = require('./foxtel.com.au.config.js') +const fs = require('fs') +const path = require('path') +const dayjs = require('dayjs') +const utc = require('dayjs/plugin/utc') +const customParseFormat = require('dayjs/plugin/customParseFormat') +dayjs.extend(customParseFormat) +dayjs.extend(utc) + +const date = dayjs.utc('2022-11-08', 'YYYY-MM-DD').startOf('d') +const channel = { + site_id: 'Channel-9-Sydney/NIN', + xmltv_id: 'Channel9Sydney.au' +} + +it('can generate valid url', () => { + expect(url({ channel, date })).toBe( + 'https://www.foxtel.com.au/tv-guide/channel/Channel-9-Sydney/NIN/2022/11/08' + ) +}) + +it('can generate valid request headers', () => { + expect(request.headers).toMatchObject({ + 'Accept-Language': 'en-US,en;', + Cookie: 'AAMC_foxtel_0=REGION|6' + }) +}) + +it('can parse response', () => { + const content = fs.readFileSync(path.resolve(__dirname, '__data__/content.html')) + + let results = parser({ content }) + results = results.map(p => { + p.start = p.start.toJSON() + p.stop = p.stop.toJSON() + return p + }) + + expect(results[0]).toMatchObject({ + start: '2022-11-07T12:40:00.000Z', + stop: '2022-11-07T13:30:00.000Z', + title: 'The Equalizer', + sub_title: 'Glory', + icon: 'https://images1.resources.foxtel.com.au/store2/mount1/16/3/69e0v.jpg?maxheight=90&limit=91aa1c7a2c485aeeba0706941f79f111adb35830', + rating: { + system: 'ACB', + value: 'M' + }, + season: 1, + episode: 2 + }) +}) + +it('can handle empty guide', () => { + const result = parser({ + content: fs.readFileSync(path.resolve(__dirname, '__data__/no-content.html')) + }) + expect(result).toMatchObject([]) +}) From 12c71f7347c2831340e38bfd003c23e2e9e688a0 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 7 Nov 2022 21:12:20 +0300 Subject: [PATCH 3/6] Create foxtel.com.au.config.js --- sites/foxtel.com.au/foxtel.com.au.config.js | 134 ++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 sites/foxtel.com.au/foxtel.com.au.config.js diff --git a/sites/foxtel.com.au/foxtel.com.au.config.js b/sites/foxtel.com.au/foxtel.com.au.config.js new file mode 100644 index 00000000..1f5ee20a --- /dev/null +++ b/sites/foxtel.com.au/foxtel.com.au.config.js @@ -0,0 +1,134 @@ +const axios = require('axios') +const dayjs = require('dayjs') +const cheerio = require('cheerio') + +module.exports = { + ignore: true, // returns "Access Denied" or nothing + site: 'foxtel.com.au', + url({ channel, date }) { + return `https://www.foxtel.com.au/tv-guide/channel/${channel.site_id}/${date.format( + 'YYYY/MM/DD' + )}` + }, + request: { + headers: { + 'Accept-Language': 'en-US,en;', + Cookie: 'AAMC_foxtel_0=REGION|6' + } + }, + parser: function ({ content }) { + let programs = [] + const items = parseItems(content) + for (let item of items) { + const $item = cheerio.load(item) + const prev = programs[programs.length - 1] + const start = parseStart($item) + if (prev) { + if (start.isBefore(prev.start)) { + start = start.add(1, 'd') + date = date.add(1, 'd') + } + prev.stop = start + } + const stop = start.add(30, 'm') + programs.push({ + title: parseTitle($item), + sub_title: parseSubTitle($item), + icon: parseIcon($item), + rating: parseRating($item), + season: parseSeason($item), + episode: parseEpisode($item), + start, + stop + }) + } + + return programs + }, + async channels() { + const data = await axios + .get(`https://www.foxtel.com.au/webepg/ws/foxtel/channels?regionId=8336`, { + headers: { + 'Accept-Language': 'en-US,en;', + Cookie: 'AAMC_foxtel_0=REGION|6' + } + }) + .then(r => r.data) + .catch(console.log) + + return data.channels.forEach(item => { + let name = item.name.replace(/\+/g, '-').replace(/\&/g, '') + const slug = name.replace(/[^a-z0-9\s]/gi, '').replace(/[^a-z0-9]/i, '-') + + return { + name: item.name.replace(/\&/g, '&'), + site_id: `${slug}/${item.channelTag}` + } + }) + } +} + +function parseSeason($item) { + let seasonString = $item('.epg-event-description > div > abbr:nth-child(1)').attr('title') + if (!seasonString) return null + let [_, season] = seasonString.match(/^Season: (\d+)/) || [null, null] + + return season ? parseInt(season) : null +} + +function parseEpisode($item) { + let episodeString = $item('.epg-event-description > div > abbr:nth-child(2)').attr('title') + if (!episodeString) return null + let [_, episode] = episodeString.match(/^Episode: (\d+)/) || [null, null] + + return episode ? parseInt(episode) : null +} + +function parseIcon($item) { + return $item('.epg-event-thumbnail > img').attr('src') +} + +function parseTitle($item) { + return $item('.epg-event-description').clone().children().remove().end().text().trim() +} + +function parseSubTitle($item) { + let subtitle = $item('.epg-event-description > div') + .clone() + .children() + .remove() + .end() + .text() + .trim() + .split(',') + + subtitle = subtitle.pop() + const [_, rating] = subtitle.match(/\(([^)]+)\)$/) || [null, null] + + return subtitle.replace(`(${rating})`, '').trim() +} + +function parseRating($item) { + const subtitle = $item('.epg-event-description > div').text().trim() + const [_, rating] = subtitle.match(/\(([^)]+)\)$/) || [null, null] + + return rating + ? { + system: 'ACB', + value: rating + } + : null +} + +function parseStart($item) { + const unix = $item('*').data('scheduled-date') + + return dayjs(parseInt(unix)) +} + +function parseItems(content) { + if (!content) return [] + const $ = cheerio.load(content) + + return $(`#epg-channel-events > a`).toArray() +} From 5e90ac52960f39072c1548aeb7fa408da220fe23 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 7 Nov 2022 21:12:26 +0300 Subject: [PATCH 4/6] Create foxtel.com.au_au.channels.xml --- .../foxtel.com.au_au.channels.xml | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 sites/foxtel.com.au/foxtel.com.au_au.channels.xml diff --git a/sites/foxtel.com.au/foxtel.com.au_au.channels.xml b/sites/foxtel.com.au/foxtel.com.au_au.channels.xml new file mode 100644 index 00000000..96fa0829 --- /dev/null +++ b/sites/foxtel.com.au/foxtel.com.au_au.channels.xml @@ -0,0 +1,197 @@ + + + + + + \ No newline at end of file From a8d381cd2bbfd611a055cb3ba3046f6d05bc6597 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 7 Nov 2022 21:12:30 +0300 Subject: [PATCH 5/6] Create foxtel.com.au.yml --- .github/workflows/foxtel.com.au.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/foxtel.com.au.yml diff --git a/.github/workflows/foxtel.com.au.yml b/.github/workflows/foxtel.com.au.yml new file mode 100644 index 00000000..b3688e19 --- /dev/null +++ b/.github/workflows/foxtel.com.au.yml @@ -0,0 +1,17 @@ +name: foxtel.com.au +on: + schedule: + - cron: '0 3 * * *' + workflow_dispatch: + workflow_run: + workflows: [_trigger] + types: + - completed +jobs: + load: + uses: ./.github/workflows/_load.yml + with: + site: ${{github.workflow}} + secrets: + APP_ID: ${{ secrets.APP_ID }} + APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }} From ad84cffd8509efe40dc8ca4049b54d57321a7373 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 7 Nov 2022 21:16:42 +0300 Subject: [PATCH 6/6] Update lint.js --- scripts/commands/channels/lint.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/commands/channels/lint.js b/scripts/commands/channels/lint.js index 79e0ca18..cdcc2631 100644 --- a/scripts/commands/channels/lint.js +++ b/scripts/commands/channels/lint.js @@ -16,7 +16,7 @@ const xsd = ` - +