mirror of
https://github.com/iptv-org/epg
synced 2026-09-10 22:06:56 -04:00
Update tv.mail.ru Api guide and channels.
Signed-off-by: Toha <tohenk@yahoo.com>
This commit is contained in:
Toha
@@ -1,126 +1,245 @@
|
||||
const axios = require('axios')
|
||||
const cheerio = require('cheerio')
|
||||
const crypto = require('crypto')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const timezone = require('dayjs/plugin/timezone')
|
||||
const axios = require('axios')
|
||||
const uniqBy = require('lodash.uniqby')
|
||||
const doFetch = require('@ntlab/sfetch')
|
||||
|
||||
dayjs.extend(utc)
|
||||
dayjs.extend(timezone)
|
||||
|
||||
const eventIds = []
|
||||
const caches = {}
|
||||
const headers = {
|
||||
'user-agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36',
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
site: 'tv.mail.ru',
|
||||
days: 2,
|
||||
delay: 1000,
|
||||
url({ channel, date }) {
|
||||
return `https://tv.mail.ru/ajax/channel/?region_id=70&channel_id=${
|
||||
channel.site_id
|
||||
}&date=${date.format('YYYY-MM-DD')}`
|
||||
request: {
|
||||
cache: {
|
||||
ttl: 24 * 60 * 60 * 1000 // 1 day
|
||||
},
|
||||
headers
|
||||
},
|
||||
parser({ content, date }) {
|
||||
async url({ date }) {
|
||||
return await followUrl(getUrl(date))
|
||||
},
|
||||
async parser({ content, channel, date }) {
|
||||
const programs = []
|
||||
const items = parseItems(content)
|
||||
items.forEach(item => {
|
||||
const prev = programs[programs.length - 1]
|
||||
let start = parseStart(item, date)
|
||||
if (prev) {
|
||||
if (start < prev.start) {
|
||||
start = start.add(1, 'day')
|
||||
date = date.add(1, 'day')
|
||||
if (content) {
|
||||
// common utility
|
||||
const v = { e: [], d: date, p: 1 }
|
||||
const ev = (res, url) => {
|
||||
if (typeof res === 'string' || Buffer.isBuffer(res)) {
|
||||
res = JSON.parse(res)
|
||||
}
|
||||
if (Array.isArray(res?.data?.schedule?.items)) {
|
||||
res.data.schedule.items
|
||||
.filter(schedule => schedule.id == channel.site_id)
|
||||
.forEach(schedule => {
|
||||
if (Array.isArray(schedule.events)) {
|
||||
schedule.events
|
||||
.filter(event => !eventIds.includes(event.id))
|
||||
.forEach(event => {
|
||||
// use event id as unique filter
|
||||
if (!v.e.find(item => item.id === event.id)) {
|
||||
v.e.push(event)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
// cache result
|
||||
if (url) {
|
||||
caches[url] = res
|
||||
}
|
||||
}
|
||||
prev.stop = start
|
||||
}
|
||||
const stop = start.add(1, 'hour')
|
||||
programs.push({
|
||||
title: item.name,
|
||||
category: parseCategory(item),
|
||||
start,
|
||||
stop
|
||||
const mu = (next = true, dt = null) => {
|
||||
return getUrl(dt ? dt : v.d, next ? ++v.p : v.p)
|
||||
}
|
||||
const mq = oUrl => ({
|
||||
async url(queue) {
|
||||
return await followUrl(queue.oUrl)
|
||||
},
|
||||
oUrl,
|
||||
params: module.exports.request
|
||||
})
|
||||
})
|
||||
const cr = async (next = true, dt = null) => {
|
||||
const oUrl = mu(next, dt)
|
||||
if (caches[oUrl]) {
|
||||
ev(caches[oUrl])
|
||||
} else {
|
||||
await doFetch([mq(oUrl)], (queue, res) => {
|
||||
ev(res, queue.oUrl)
|
||||
})
|
||||
}
|
||||
}
|
||||
// add main content
|
||||
ev(content)
|
||||
// fetch next page until site is found
|
||||
await new Promise(resolve => {
|
||||
const f = async () => {
|
||||
if (v.e.length) {
|
||||
resolve()
|
||||
} else {
|
||||
await cr()
|
||||
f()
|
||||
}
|
||||
}
|
||||
f()
|
||||
})
|
||||
if (v.e.length) {
|
||||
date = date.startOf('d')
|
||||
v.n = [...v.e]
|
||||
v.e = []
|
||||
// fetch previous day
|
||||
await cr(false, date.subtract(1, 'd'))
|
||||
v.e.push(...v.n)
|
||||
// process collected events
|
||||
const queues = []
|
||||
const events = []
|
||||
v.e
|
||||
.forEach(event => {
|
||||
const start = dayjs.utc(event.start_ts * 1000)
|
||||
const stop = dayjs.utc(event.stop_ts * 1000)
|
||||
if (date.isSame(start, 'd') || (date.isSame(stop, 'd') && stop > date)) {
|
||||
eventIds.push(event.id)
|
||||
const url = `https://tv.mail.ru${event.url}`
|
||||
const evt = {
|
||||
program: {
|
||||
title: event.name,
|
||||
image: event.image,
|
||||
start,
|
||||
stop
|
||||
}
|
||||
}
|
||||
if (caches[url]) {
|
||||
evt.detail = caches[url]
|
||||
} else {
|
||||
queues.push({
|
||||
async url(queue) {
|
||||
return await followUrl(queue.oUrl)
|
||||
},
|
||||
oUrl: url,
|
||||
params: module.exports.request,
|
||||
evt
|
||||
})
|
||||
}
|
||||
events.push(evt)
|
||||
}
|
||||
})
|
||||
// fetch detailed programs
|
||||
if (process.env.EPG_DETAILED_GUIDE) {
|
||||
await doFetch(queues, (queue, res) => {
|
||||
queue.evt.detail = res
|
||||
caches[queue.oUrl] = res
|
||||
})
|
||||
}
|
||||
// add programs
|
||||
programs.push(...events
|
||||
.map(evt => ({
|
||||
...evt.program,
|
||||
...parseProgram(evt.detail)
|
||||
})
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return programs
|
||||
},
|
||||
async channels() {
|
||||
const regions = [5506, 1096, 1125, 285]
|
||||
|
||||
let channels = []
|
||||
for (let region of regions) {
|
||||
const totalPages = await getTotalPageCount(region)
|
||||
const pages = Array.from(Array(totalPages).keys())
|
||||
for (let page of pages) {
|
||||
const data = await axios
|
||||
.get('https://tv.mail.ru/ajax/channel/list/', {
|
||||
params: { page },
|
||||
headers: {
|
||||
cookie: `s=fver=0|geo=${region};`
|
||||
}
|
||||
})
|
||||
.then(r => r.data)
|
||||
.catch(console.log)
|
||||
|
||||
data.channels.forEach(item => {
|
||||
channels.push({
|
||||
const channels = []
|
||||
const args = { date: dayjs(), page: 0 }
|
||||
const queues = [getUrl(args.date, ++args.page)]
|
||||
await doFetch(queues, (url, res) => {
|
||||
if (res?.data?.schedule?.has_next_page) {
|
||||
queues.push(getUrl(args.date, ++args.page))
|
||||
}
|
||||
if (Array.isArray(res?.data?.schedule?.items)) {
|
||||
channels.push(...res.data.schedule.items
|
||||
.map(item => ({
|
||||
lang: 'ru',
|
||||
name: item.name,
|
||||
site_id: item.id
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return uniqBy(channels, 'site_id')
|
||||
}
|
||||
}
|
||||
|
||||
async function getTotalPageCount(region) {
|
||||
const data = await axios
|
||||
.get('https://tv.mail.ru/ajax/channel/list/', {
|
||||
params: { page: 0 },
|
||||
headers: {
|
||||
cookie: `s=fver=0|geo=${region};`
|
||||
}))
|
||||
)
|
||||
}
|
||||
})
|
||||
.then(r => r.data)
|
||||
.catch(console.log)
|
||||
|
||||
return data.total
|
||||
return channels
|
||||
}
|
||||
}
|
||||
|
||||
function parseStart(item, date) {
|
||||
const dateString = `${date.format('YYYY-MM-DD')} ${item.start}`
|
||||
|
||||
return dayjs.tz(dateString, 'YYYY-MM-DD HH:mm', 'Europe/Moscow').utc()
|
||||
}
|
||||
|
||||
function parseCategory(item) {
|
||||
const categories = {
|
||||
1: 'Фильм',
|
||||
2: 'Сериал',
|
||||
6: 'Документальное',
|
||||
7: 'Телемагазин',
|
||||
8: 'Позновательное',
|
||||
10: 'Другое',
|
||||
14: 'ТВ-шоу',
|
||||
16: 'Досуг,Хобби',
|
||||
17: 'Ток-шоу',
|
||||
18: 'Юмористическое',
|
||||
23: 'Музыка',
|
||||
24: 'Развлекательное',
|
||||
25: 'Игровое',
|
||||
26: 'Новости'
|
||||
function parseProgram(data) {
|
||||
const res = {}
|
||||
if (data) {
|
||||
const props = {
|
||||
title: '.fa7d86047b [data-qa="Title"]',
|
||||
sub_title: '.d4a57e7163',
|
||||
category: '[data-qa="TagLink"] [data-qa="Text"]',
|
||||
description: ['.f9cbc7a035', 'h2'],
|
||||
actor: '[data-qa*="TEST_IDS__EVENT_ATTR__actors_link_"]',
|
||||
director: '[data-qa*="TEST_IDS__EVENT_ATTR__director_link_"]',
|
||||
date: '[data-qa="TEST_IDS__TEVENT_META__start_year"]',
|
||||
country: '.ec1bc00977'
|
||||
}
|
||||
const $ = cheerio.load(data)
|
||||
for (const [prop, selector] of Object.entries(props)) {
|
||||
const item = $(Array.isArray(selector) ? selector[0] : selector)
|
||||
if (item.length) {
|
||||
// prune children
|
||||
if (Array.isArray(selector)) {
|
||||
item.find(selector[1]).remove()
|
||||
}
|
||||
if (item.length > 1) {
|
||||
res[prop] = item.toArray()
|
||||
.map(item => $(item).text().trim())
|
||||
} else {
|
||||
res[prop] = item.text().trim()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return categories[item.category_id]
|
||||
? {
|
||||
lang: 'ru',
|
||||
value: categories[item.category_id]
|
||||
}
|
||||
: null
|
||||
return res
|
||||
}
|
||||
|
||||
function parseItems(content) {
|
||||
const json = JSON.parse(content)
|
||||
if (!Array.isArray(json.schedule) || !json.schedule[0]) return []
|
||||
const event = json.schedule[0].event || []
|
||||
function getUrl(date, page = 1) {
|
||||
page = page > 1 ? `&page=${page}` : ''
|
||||
return `https://tv.mail.ru/ajax/service/index/schedule/?region_id=70&channel_type=all&date=${
|
||||
date.format('YYYY-MM-DD')
|
||||
}&appearance=grid&period=all${
|
||||
page
|
||||
}`
|
||||
}
|
||||
|
||||
return [...event.past, ...event.current]
|
||||
async function followUrl(url) {
|
||||
const f = async (m = 'head') => {
|
||||
await axios[m](url, { headers })
|
||||
.then(res => {
|
||||
if (res.headers && res.headers['x-challenge'] === 'required') {
|
||||
const challengeUrl = new URL(res.request.res.responseUrl)
|
||||
if (challengeUrl.searchParams.has('hash429')) {
|
||||
const key = crypto.hash('md5', challengeUrl.searchParams.get('hash429'))
|
||||
if (challengeUrl.searchParams.has('key')) {
|
||||
challengeUrl.searchParams.set('key', key)
|
||||
} else {
|
||||
challengeUrl.searchParams.append('key', key)
|
||||
}
|
||||
url = challengeUrl.toString()
|
||||
} else {
|
||||
throw new Error('Unknown challenge page!')
|
||||
}
|
||||
}
|
||||
})
|
||||
// @todo: handle captcha on 429
|
||||
.catch(console.error)
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
return await f()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user