Move all functions to scripts folder

This commit is contained in:
freearhey
2019-11-02 12:45:09 +03:00
parent b4bb40dacd
commit 6b36d8d5a2
6 changed files with 176 additions and 175 deletions

174
scripts/format.js Normal file
View File

@@ -0,0 +1,174 @@
const helper = require('./helper')
const debug = false
const verbose = false
const parseEpg = process.env.npm_config_epg || false
let stats = {
total: 0,
updated: 0,
duplicates: 0,
unvalid: 0,
removed: 0
}
let buffer = {}
let unsorted = {}
async function main() {
console.log(`Parsing 'index.m3u'...`)
const playlist = helper.parsePlaylist('index.m3u')
const countries = playlist.items
if(debug) {
console.log('Debug mode is turn on')
}
const unsortedPlaylist = helper.parsePlaylist('channels/unsorted.m3u')
for(const item of unsortedPlaylist.items) {
unsorted[item.url] = helper.createChannel(item)
}
for(let country of countries) {
if (helper.skipPlaylist(country.url)) {
continue
}
if(verbose) {
console.log(`Clear cache...`)
}
helper.clearCache()
console.log(`Parsing '${country.url}'...`)
const playlist = helper.parsePlaylist(country.url)
if(verbose) {
console.log(`Creating channels list...`)
}
let channels = []
for(let item of playlist.items) {
let channel = helper.createChannel(item)
if(helper.checkCache(channel.url)) {
stats.duplicates++
} else if(!helper.validateUrl(channel.url)) {
stats.unvalid++
} else {
channels.push(channel)
helper.addToCache(channel.url)
}
if(unsorted[channel.url]) {
if(verbose) {
console.log(`Removed '${channel.url}' from 'channels/unsorted.m3u'...`)
}
delete unsorted[channel.url]
stats.removed++
stats.duplicates++
}
}
const epgUrl = playlist.header.attrs['x-tvg-url']
if(epgUrl && !buffer[epgUrl] && parseEpg) {
try {
console.log(`Loading '${epgUrl}'...`)
const epg = await helper.loadEPG(epgUrl)
console.log(`Adding '${epgUrl}' to buffer...`)
buffer[epgUrl] = epg
} catch(e) {
console.log(`Could not load '${epgUrl}'`)
}
}
if(buffer[epgUrl]) {
console.log('Add missing tvg-id from EPG by channel title...')
for(let channel of channels) {
for(let channelId in buffer[epgUrl].channels) {
let c = buffer[epgUrl].channels[channelId]
for(let epgName of c.name) {
if(epgName.value) {
let escaped = helper.escapeStringRegexp(epgName.value)
channelTitle = channel.title.replace(/(fhd|hd|sd|高清)$/i, '').trim()
let regexp = new RegExp(`^${escaped}$`, 'i')
if(regexp.test(channelTitle)) {
if(!channel.id) {
channel.id = c.id
continue
}
}
}
}
}
}
}
if(buffer[epgUrl]) {
console.log(`Fills in missing channel's data...`)
for(let channel of channels) {
let channelId = channel.id
if(!channelId) continue
let c = buffer[epgUrl].channels[channelId]
if(!c) continue
let updated = false
if(!channel.name && c.name.length) {
channel.name = c.name[0].value
updated = true
if(verbose) {
console.log(`Added name '${c.name[0].value}' to '${channel.id}'`)
}
}
if(!channel.language && c.name.length && c.name[0].lang) {
let language = helper.getISO6391Name(c.name[0].lang)
channel.language = language
updated = true
if(verbose) {
console.log(`Added language '${language}' to '${channel.id}'`)
}
}
if(!channel.logo && c.icon.length) {
const icon = c.icon[0].split('|')[0]
channel.logo = icon
updated = true
if(verbose) {
console.log(`Added logo '${icon}' to '${channel.id}'`)
}
}
if(updated) {
stats.updated++
}
}
}
if(verbose) {
console.log(`Sorting channels...`)
}
channels = helper.sortByTitleAndUrl(channels)
if(!debug) {
console.log(`Updating '${country.url}'...`)
helper.createFile(country.url, playlist.getHeader())
channels.forEach(channel => {
helper.appendToFile(country.url, channel.toString())
})
}
stats.total += channels.length
}
if(!debug & stats.removed > 0) {
console.log(`Updating 'channels/unsorted.m3u'...`)
helper.createFile('channels/unsorted.m3u', playlist.getHeader())
Object.values(unsorted).forEach(channel => {
helper.appendToFile('channels/unsorted.m3u', channel.toString())
})
}
console.log(`Total: ${stats.total}. Duplicates: ${stats.duplicates}. Unvalid: ${stats.unvalid}. Updated: ${stats.updated}.`)
}
main()

164
scripts/generate.js Normal file
View File

@@ -0,0 +1,164 @@
const helper = require('./helper')
let list = {
all: [],
countries: {},
languages: {},
categories: {}
}
function main() {
console.log(`Parsing index...`)
parseIndex()
console.log('Generating index.country.m3u...')
generateCountryIndex()
console.log('Generating index.language.m3u...')
generateLanguageIndex()
console.log('Generating index.content.m3u...')
generateContentIndex()
console.log('Generating index.full.m3u...')
generateFullIndex()
console.log('Generating /categories...')
generateCategories()
console.log('Generating /languages...')
generateLanguages()
console.log('Done.\n')
console.log(`Countries: ${Object.values(list.countries).length}. Languages: ${Object.values(list.languages).length}. Categories: ${Object.values(list.categories).length}. Channels: ${list.all.length}.`)
}
function parseIndex() {
const root = helper.parsePlaylist('index.m3u')
let countries = {}
let languages = {}
let categories = {}
for(let rootItem of root.items) {
const playlist = helper.parsePlaylist(rootItem.url)
const countryCode = helper.getBasename(rootItem.url).toLowerCase()
const countryName = rootItem.name
for(let item of playlist.items) {
const channel = helper.createChannel(item)
channel.countryCode = countryCode
channel.countryName = countryName
// all
list.all.push(channel)
// country
if(!countries[countryCode]) {
countries[countryCode] = []
}
countries[countryCode].push(channel)
// language
const languageCode = helper.getISO6391Code(channel.language) || 'undefined'
if(!languages[languageCode]) {
languages[languageCode] = []
}
languages[languageCode].push(channel)
// category
const categoryCode = channel.group.toLowerCase() || 'other'
if(!categories[categoryCode]) {
categories[categoryCode] = []
}
categories[categoryCode].push(channel)
}
}
list.countries = countries
list.languages = languages
list.categories = categories
}
function generateCountryIndex() {
const filename = `index.country.m3u`
helper.createFile(filename, '#EXTM3U\n')
for(let channel of list.all) {
const group = channel.group
channel.group = channel.countryName
helper.appendToFile(filename, channel.toString())
channel.group = group
}
}
function generateLanguageIndex() {
const filename = `index.language.m3u`
helper.createFile(filename, '#EXTM3U\n')
const channels = list.all.sort((a, b) => {
if(a.language < b.language) { return -1 }
if(a.language > b.language) { return 1 }
return 0
})
for(let channel of channels) {
const group = channel.group
channel.group = channel.language
helper.appendToFile(filename, channel.toString())
channel.group = group
}
}
function generateContentIndex() {
const filename = `index.content.m3u`
helper.createFile(filename, '#EXTM3U\n')
const channels = list.all.sort((a, b) => {
if(a.group < b.group) { return -1 }
if(a.group > b.group) { return 1 }
return 0
})
for(let channel of channels) {
helper.appendToFile(filename, channel.toString())
}
}
function generateFullIndex() {
const filename = `index.full.m3u`
helper.createFile(filename, '#EXTM3U\n')
const channels = list.all.sort((a, b) => {
if(a.countryName < b.countryName) { return -1 }
if(a.countryName > b.countryName) { return 1 }
if(a.group < b.group) { return -1 }
if(a.group > b.group) { return 1 }
return 0
})
for(let channel of channels) {
const group = channel.group
channel.group = [ channel.countryName, channel.group ].filter(i => i).join(';')
helper.appendToFile(filename, channel.toString())
channel.group = group
}
}
function generateCategories() {
for(let cid in list.categories) {
let category = list.categories[cid]
const filename = `categories/${cid}.m3u`
helper.createFile(filename, '#EXTM3U\n')
for(let channel of category) {
helper.appendToFile(filename, channel.toString())
}
}
}
function generateLanguages() {
for(let lid in list.languages) {
let language = list.languages[lid]
const filename = `languages/${lid}.m3u`
helper.createFile(filename, '#EXTM3U\n')
for(let channel of language) {
helper.appendToFile(filename, channel.toString())
}
}
}
main()

266
scripts/helper.js Normal file
View File

@@ -0,0 +1,266 @@
const fs = require("fs")
const path = require('path')
const parser = require('iptv-playlist-parser')
const axios = require('axios')
const zlib = require("zlib")
const epgParser = require('epg-parser')
const urlParser = require('url')
const ISO6391 = require('iso-639-1')
const escapeStringRegexp = require('escape-string-regexp')
const markdownInclude = require('markdown-include')
let cache = {}
let helper = {}
helper.compileMarkdown = function(filepath) {
return markdownInclude.compileFiles(path.resolve(__dirname, filepath))
}
helper.escapeStringRegexp = function(scring) {
return escapeStringRegexp(string)
}
helper.getISO6391Name = function(code) {
return ISO6391.getName(code)
}
helper.getISO6391Code = function(name) {
return ISO6391.getCode(name)
}
helper.parsePlaylist = function(filename) {
const content = readFile(filename)
const result = parser.parse(content)
return new Playlist(result)
}
helper.createChannel = function(data) {
return new Channel({
id: data.tvg.id,
name: data.tvg.name,
language: data.tvg.language,
logo: data.tvg.logo,
group: data.group.title,
url: data.url,
title: data.name
})
}
helper.loadEPG = async function(url) {
const content = await getEPGFile(url)
const result = epgParser.parse(content)
const channels = {}
for(let channel of result.channels) {
channels[channel.id] = channel
}
return Promise.resolve({
url,
channels
})
}
helper.getEPGFile = function(url) {
return new Promise((resolve, reject) => {
var buffer = []
axios({
method: 'get',
url: url,
responseType:'stream'
}).then(res => {
let stream
if(/\.gz$/i.test(url)) {
let gunzip = zlib.createGunzip()
res.data.pipe(gunzip)
stream = gunzip
} else {
stream = res.data
}
stream.on('data', function(data) {
buffer.push(data.toString())
}).on("end", function() {
resolve(buffer.join(""))
}).on("error", function(e) {
reject(e)
})
}).catch(e => {
reject(e)
})
})
}
helper.sortByTitleAndUrl = function(arr) {
return arr.sort(function(a, b) {
var titleA = a.title.toLowerCase()
var titleB = b.title.toLowerCase()
var urlA = a.url.toLowerCase()
var urlB = b.url.toLowerCase()
if(titleA < titleB) return -1
if(titleA > titleB) return 1
if(urlA < urlB) return -1
if(urlA > urlB) return 1
return 0
})
}
helper.readFile = function(filename) {
return fs.readFileSync(path.resolve(__dirname) + `/../${filename}`, { encoding: "utf8" })
}
helper.appendToFile = function(filename, data) {
fs.appendFileSync(path.resolve(__dirname) + '/../' + filename, data)
}
helper.createFile = function(filename, data) {
fs.writeFileSync(path.resolve(__dirname) + '/../' + filename, data)
}
helper.getBasename = function(filename) {
return path.basename(filename, path.extname(filename))
}
helper.addToCache = function(url) {
let id = getUrlPath(url)
cache[id] = true
}
helper.checkCache = function(url) {
let id = getUrlPath(url)
return cache.hasOwnProperty(id)
}
helper.clearCache = function() {
cache = {}
}
helper.getUrlPath = function(u) {
let parsed = urlParser.parse(u)
let searchQuery = parsed.search || ''
let path = parsed.host + parsed.pathname + searchQuery
return path.toLowerCase()
}
helper.validateUrl = function(channelUrl) {
const url = new URL(channelUrl)
const host = url.hostname
const blacklist = [
'80.80.160.168', // repeats on a loop
'63.237.48.3', // not a live stream
'189.216.247.113', // not working streams
]
return blacklist.indexOf(host) === -1
}
helper.skipPlaylist = function(filename) {
let testCountry = process.env.npm_config_country
let excludeList = process.env.npm_config_exclude
let excludeCountries = excludeList ? excludeList.split(',') : []
if (testCountry && filename !== 'channels/' + testCountry + '.m3u') return true
for(const countryCode of excludeCountries) {
if (filename === 'channels/' + countryCode + '.m3u') return true
}
return false
}
helper.generateTable = function(data, options) {
let output = '<table>'
output += '<thead><tr>'
for (let column of options.columns) {
output += `<th align="${column.align}">${column.name}</th>`
}
output += '</tr></thead>'
output += '<tbody>'
for (let item of data) {
output += '<tr>'
let i = 0
for (let prop in item) {
const column = options.columns[i]
let nowrap = column.nowrap
let align = column.align
output += `<td align="${align}"${nowrap ? ' nowrap' : ''}>${item[prop]}</td>`
i++
}
output += '</tr>'
}
output += '</tbody>'
output += '</table>'
return output
}
class Playlist {
constructor(data) {
this.header = data.header
this.items = data.items
}
getHeader() {
let parts = ['#EXTM3U']
for(let key in this.header.attrs) {
let value = this.header.attrs[key]
if(value) {
parts.push(`${key}="${value}"`)
}
}
return `${parts.join(' ')}\n`
}
}
class Channel {
constructor(data) {
this.id = data.id
this.name = data.name
this.language = this._filterLanguage(data.language)
this.logo = data.logo
this.group = this._filterGroup(data.group)
this.url = data.url
this.title = data.title
}
_filterGroup(groupTitle) {
if(!groupTitle) return ''
const supportedCategories = [ 'Auto','Business', 'Classic','Comedy','Documentary','Education','Entertainment', 'Family','Fashion','Food', 'General', 'Health', 'History', 'Hobby', 'Kids', 'Legislative','Lifestyle','Local', 'Movies', 'Music', 'News', 'Quiz', 'Religious','Sci-Fi', 'Shop', 'Sport', 'Travel', 'Weather', 'XXX' ]
const groupIndex = supportedCategories.map(g => g.toLowerCase()).indexOf(groupTitle.toLowerCase())
if(groupIndex === -1) {
groupTitle = ''
} else {
groupTitle = supportedCategories[groupIndex]
}
return groupTitle
}
_filterLanguage(languageName) {
if(ISO6391.getCode(languageName) !== '') {
return languageName
}
return ''
}
toString() {
const info = `-1 tvg-id="${this.id}" tvg-name="${this.name}" tvg-language="${this.language}" tvg-logo="${this.logo}" group-title="${this.group}",${this.title}`
return '#EXTINF:' + info + '\n' + this.url + '\n'
}
}
module.exports = helper

132
scripts/update-readme.js Normal file
View File

@@ -0,0 +1,132 @@
const helper = require('./helper')
let output = {
countries: [],
languages: [],
categories: []
}
function main() {
console.log(`Parsing index...`)
parseIndex()
console.log(`Generating countries table...`)
generateCountriesTable()
console.log(`Generating languages table...`)
generateLanguagesTable()
console.log(`Generating categories table...`)
generateCategoriesTable()
console.log(`Generating README.md...`)
generateReadme()
console.log(`Done.`)
}
function parseIndex() {
const root = helper.parsePlaylist('index.m3u')
let languages = {}
let categories = {}
for(let rootItem of root.items) {
const playlist = helper.parsePlaylist(rootItem.url)
const countryCode = helper.getBasename(rootItem.url).toUpperCase()
const epg = playlist.header.attrs['x-tvg-url'] ? `<code>${playlist.header.attrs['x-tvg-url']}</code>` : ''
// country
output.countries.push({
country: rootItem.name,
channels: playlist.items.length,
playlist: `<code>https://iptv-org.github.io/iptv/${rootItem.url}</code>`,
epg
})
for(let item of playlist.items) {
// language
const languageName = item.tvg.language || 'Undefined'
const languageCode = helper.getISO6391Code(languageName) || 'undefined'
if(languages[languageCode]) {
languages[languageCode].channels++
} else {
languages[languageCode] = {
language: languageName,
channels: 1,
playlist: `<code>https://iptv-org.github.io/iptv/languages/${languageCode}.m3u</code>`
}
}
// category
const categoryName = item.group.title || 'Other'
const categoryCode = categoryName.toLowerCase()
if(categories[categoryCode]) {
categories[categoryCode].channels++
} else {
categories[categoryCode] = {
category: categoryName,
channels: 1,
playlist: `<code>https://iptv-org.github.io/iptv/categories/${categoryCode}.m3u</code>`
}
}
}
}
output.languages = Object.values(languages)
output.categories = Object.values(categories)
}
function generateCountriesTable() {
const table = helper.generateTable(output.countries, {
columns: [
{ name: 'Country', align: 'left' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', align: 'left', nowrap: true },
{ name: 'EPG', align: 'left' }
]
})
helper.createFile('./.readme/_countries.md', table)
}
function generateLanguagesTable() {
output.languages.sort((a, b) => {
if(a.language === 'Undefined') { return 1 }
if(b.language === 'Undefined') { return -1 }
if(a.language < b.language) { return -1 }
if(a.language > b.language) { return 1 }
return 0
})
const table = helper.generateTable(output.languages, {
columns: [
{ name: 'Language', align: 'left' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', align: 'left' }
]
})
helper.createFile('./.readme/_languages.md', table)
}
function generateCategoriesTable() {
output.categories.sort((a, b) => {
if(a.category === 'Other') { return 1 }
if(b.category === 'Other') { return -1 }
if(a.category < b.category) { return -1 }
if(a.category > b.category) { return 1 }
return 0
})
const table = helper.generateTable(output.categories, {
columns: [
{ name: 'Category', align: 'left' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', align: 'left' }
]
})
helper.createFile('./.readme/_categories.md', table)
}
function generateReadme() {
helper.compileMarkdown('../.readme/config.json')
}
main()