epg/sites/tv.mail.ru/tv.mail.ru.config.js

const axios = require('axios')
const cheerio = require('cheerio')
const crypto = require('crypto')
const dayjs = require('dayjs')
const utc = require('dayjs/plugin/utc')
const doFetch = require('@ntlab/sfetch')

dayjs.extend(utc)

const eventIds = []
const caches = {}
const headers = {
  'user-agent':
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36',
}

module.exports = {
  site: 'tv.mail.ru',
  days: 2,
  request: {
    cache: {
      ttl: 24 * 60 * 60 * 1000 // 1 day
    },
    headers
  },
  async url({ date }) {
    return await followUrl(getUrl(date))
  },
  async parser({ content, channel, date }) {
    const programs = []
    if (content) {
      // common utility
      const v = { e: [], d: date, p: 1 }
      const ev = (res, url) => {
        if (typeof res === 'string' || Buffer.isBuffer(res)) {
          res = JSON.parse(res)
        }
        if (Array.isArray(res?.data?.schedule?.items)) {
          res.data.schedule.items
            .filter(schedule => schedule.id == channel.site_id)
            .forEach(schedule => {
              if (Array.isArray(schedule.events)) {
                schedule.events
                  .filter(event => !eventIds.includes(event.id))
                  .forEach(event => {
                    // use event id as unique filter
                    if (!v.e.find(item => item.id === event.id)) {
                      v.e.push(event)
                    }
                  })
              }
            })
          // cache result
          if (url) {
            caches[url] = res
          }
        }
      }
      const mu = (next = true, dt = null) => {
        return getUrl(dt ? dt : v.d, next ? ++v.p : v.p)
      }
      const mq = oUrl => ({
        async url(queue) {
          return await followUrl(queue.oUrl)
        },
        oUrl,
        params: module.exports.request
      })
      const cr = async (next = true, dt = null) => {
        const oUrl = mu(next, dt)
        if (caches[oUrl]) {
          ev(caches[oUrl])
        } else {
          await doFetch([mq(oUrl)], (queue, res) => {
            ev(res, queue.oUrl)
          })
        }
      }
      // add main content
      ev(content)
      // fetch next page until site is found
      await new Promise(resolve => {
        const f = async () => {
          if (v.e.length) {
            resolve()
          } else {
            await cr()
            f()
          }
        }
        f()
      })
      if (v.e.length) {
        date = date.startOf('d')
        v.n = [...v.e]
        v.e = []
        // fetch previous day
        await cr(false, date.subtract(1, 'd'))
        v.e.push(...v.n)
        // process collected events
        const queues = []
        const events = []
        v.e
          .forEach(event => {
            const start = dayjs.utc(event.start_ts * 1000)
            const stop = dayjs.utc(event.stop_ts * 1000)
            if (date.isSame(start, 'd') || (date.isSame(stop, 'd') && stop > date)) {
              eventIds.push(event.id)
              const url = `https://tv.mail.ru${event.url}`
              const evt = {
                program: {
                  title: event.name,
                  image: event.image,
                  start,
                  stop
                }
              }
              if (caches[url]) {
                evt.detail = caches[url]
              } else {
                queues.push({
                  async url(queue) {
                    return await followUrl(queue.oUrl)
                  },
                  oUrl: url,
                  params: module.exports.request,
                  evt
                })
              }
              events.push(evt)
            }
          })
        // fetch detailed programs
        if (process.env.EPG_DETAILED_GUIDE) {
          await doFetch(queues, (queue, res) => {
            queue.evt.detail = res
            caches[queue.oUrl] = res
          })
        }
        // add programs
        programs.push(...events
          .map(evt => ({
              ...evt.program,
              ...parseProgram(evt.detail)
            })
          )
        )
      }
    }

    return programs
  },
  async channels() {
    const channels = []
    const args = { date: dayjs(), page: 0 }
    const queues = [getUrl(args.date, ++args.page)]
    await doFetch(queues, (url, res) => {
      if (res?.data?.schedule?.has_next_page) {
        queues.push(getUrl(args.date, ++args.page))
      }
      if (Array.isArray(res?.data?.schedule?.items)) {
        channels.push(...res.data.schedule.items
          .map(item => ({
            lang: 'ru',
            name: item.name,
            site_id: item.id
          }))
        )
      }
    })

    return channels
  }
}

function parseProgram(data) {
  const res = {}
  if (data) {
    const props = {
      title: '.fa7d86047b [data-qa="Title"]',
      sub_title: '.d4a57e7163',
      category: '[data-qa="TagLink"] [data-qa="Text"]',
      description: ['.f9cbc7a035', 'h2'],
      actor: '[data-qa*="TEST_IDS__EVENT_ATTR__actors_link_"]',
      director: '[data-qa*="TEST_IDS__EVENT_ATTR__director_link_"]',
      date: '[data-qa="TEST_IDS__TEVENT_META__start_year"]',
      country: '.ec1bc00977'
    }
    const $ = cheerio.load(data)
    for (const [prop, selector] of Object.entries(props)) {
      const item = $(Array.isArray(selector) ? selector[0] : selector)
      if (item.length) {
        // prune children
        if (Array.isArray(selector)) {
          item.find(selector[1]).remove()
        }
        if (item.length > 1) {
          res[prop] = item.toArray()
            .map(item => $(item).text().trim())
        } else {
          res[prop] = item.text().trim()
        }
      }
    }
  }

  return res
}

function getUrl(date, page = 1) {
  page = page > 1 ? `&page=${page}` : ''
  return `https://tv.mail.ru/ajax/service/index/schedule/?region_id=70&channel_type=all&date=${
    date.format('YYYY-MM-DD')
  }&appearance=grid&period=all${
    page
  }`
}

async function followUrl(url) {
  const f = async (m = 'head') => {
    await axios[m](url, { headers })
      .then(res => {
        if (res.headers && res.headers['x-challenge'] === 'required') {
          const challengeUrl = new URL(res.request.res.responseUrl)
          if (challengeUrl.searchParams.has('hash429')) {
            const key = crypto.hash('md5', challengeUrl.searchParams.get('hash429'))
            if (challengeUrl.searchParams.has('key')) {
              challengeUrl.searchParams.set('key', key)
            } else {
              challengeUrl.searchParams.append('key', key)
            }
            url = challengeUrl.toString()
          } else {
            throw new Error('Unknown challenge page!')
          }
        }
      })
      // @todo: handle captcha on 429
      .catch(console.error)

    return url
  }

  return await f()
}