include node_modules so release .zip is deployable

node_modules/@extractus/feed-extractor/src/deno/cross-fetch.js

@@ -0,0 +1,2 @@
+// cross-fetch.js
+export default fetch

node_modules/@extractus/feed-extractor/src/main.js

@@ -0,0 +1,74 @@
+// main.js
+
+import { isValid as isValidUrl } from './utils/linker.js'
+
+import retrieve from './utils/retrieve.js'
+import { validate, xml2obj, isRSS, isAtom, isRdf } from './utils/xmlparser.js'
+import parseJsonFeed from './utils/parseJsonFeed.js'
+import parseRssFeed from './utils/parseRssFeed.js'
+import parseAtomFeed from './utils/parseAtomFeed.js'
+import parseRdfFeed from './utils/parseRdfFeed.js'
+
+const getopt = (options = {}) => {
+  const {
+    normalization = true,
+    descriptionMaxLen = 250,
+    useISODateFormat = true,
+    xmlParserOptions = {},
+    baseUrl = '',
+    getExtraFeedFields = () => ({}),
+    getExtraEntryFields = () => ({}),
+  } = options
+
+  return {
+    normalization,
+    descriptionMaxLen,
+    useISODateFormat,
+    xmlParserOptions,
+    baseUrl,
+    getExtraFeedFields,
+    getExtraEntryFields,
+  }
+}
+
+export const extractFromJson = (json, options = {}) => {
+  return parseJsonFeed(json, getopt(options))
+}
+
+export const extractFromXml = (xml, options = {}) => {
+  if (!validate(xml)) {
+    throw new Error('The XML document is not well-formed')
+  }
+
+  const opts = getopt(options)
+
+  const data = xml2obj(xml, opts.xmlParserOptions)
+
+  return isRSS(data)
+    ? parseRssFeed(data, opts)
+    : isAtom(data)
+      ? parseAtomFeed(data, opts)
+      : isRdf(data)
+        ? parseRdfFeed(data, opts)
+        : null
+}
+
+export const extract = async (url, options = {}, fetchOptions = {}) => {
+  if (!isValidUrl(url)) {
+    throw new Error('Input param must be a valid URL')
+  }
+
+  const data = await retrieve(url, fetchOptions)
+  if (!data.text && !data.json) {
+    throw new Error(`Failed to load content from "${url}"`)
+  }
+
+  const { type, json, text } = data
+
+  return type === 'json' ? extractFromJson(json, options) : extractFromXml(text, options)
+}
+
+export const read = async (url, options, fetchOptions) => {
+  console.warn('WARNING: read() is deprecated. Please use extract() instead!')
+  return extract(url, options, fetchOptions)
+}

node_modules/@extractus/feed-extractor/src/main.test.js

@@ -0,0 +1,505 @@
+// main.test
+/* eslint-env jest */
+
+import { readFileSync } from 'fs'
+
+import nock from 'nock'
+import { HttpsProxyAgent } from 'https-proxy-agent'
+
+import { hasProperty, isString } from 'bellajs'
+
+import { extract, extractFromXml, extractFromJson, read } from './main.js'
+import { isValid as isValidUrl } from './utils/linker.js'
+
+const env = process.env || {}
+const PROXY_SERVER = env.PROXY_SERVER || ''
+
+const feedAttrs = 'title link description generator language published entries'.split(' ')
+const entryAttrs = 'title link description published id'.split(' ')
+
+const parseUrl = (url) => {
+  const re = new URL(url)
+  return {
+    baseUrl: `${re.protocol}//${re.host}`,
+    path: re.pathname,
+  }
+}
+
+const isValidDate = (d) => {
+  return (new Date(d)).toString() !== 'Invalid Date'
+}
+
+const validateProps = (entry) => {
+  const { id, link, title, published, description } = entry
+  return isString(description) &&
+    isString(id) && id !== '' &&
+    isString(title) && title !== '' &&
+    isString(link) && isValidUrl(link) &&
+    isString(published) && isValidDate(published)
+}
+
+describe('test extract() function with common issues', () => {
+  test('extract feed from a non-string link', () => {
+    expect(extract([])).rejects.toThrow(new Error('Input param must be a valid URL'))
+  })
+
+  test('extract feed from a 404 link', () => {
+    const url = 'https://somewhere.xyz/alpha/beta'
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(404)
+    expect(extract(url)).rejects.toThrow(new Error('Request failed with error code 404'))
+  })
+
+  test('extract feed from empty xml', () => {
+    const url = 'https://empty-source.elsewhere/rss'
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, '', {
+      'Content-Type': 'application/xml',
+    })
+    expect(extract(url)).rejects.toThrow(new Error(`Failed to load content from "${url}"`))
+  })
+
+  test('extract feed from invalid xml', async () => {
+    const url = 'https://averybad-source.elsewhere/rss'
+    const xml = '<?xml version="1.0" encoding="UTF-8><noop><oops></ooops>'
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    expect(extract(url)).rejects.toThrow(new Error('The XML document is not well-formed'))
+  })
+
+  test('extract feed from invalid json', async () => {
+    const url = 'https://averybad-source.elsewhere/jsonfeed'
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, 'this is not json string', {
+      'Content-Type': 'application/json',
+    })
+    expect(extract(url)).rejects.toThrow(new Error('Failed to convert data to JSON object'))
+  })
+})
+
+describe('test extract() standard feed', () => {
+  test('extract rss feed from Google', async () => {
+    const url = 'https://some-news-page.tld/rss'
+    const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url)
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+
+  test('extract atom feed from Google', async () => {
+    const url = 'https://some-news-page.tld/atom'
+    const xml = readFileSync('test-data/atom-feed-standard-realworld.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url)
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+
+  test('extract atom feed from Google with extraFields', async () => {
+    const url = 'https://some-news-page.tld/atom'
+    const xml = readFileSync('test-data/atom-feed-standard-realworld.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      getExtraFeedFields: data => {
+        return {
+          author: data.author,
+        }
+      },
+      getExtraEntryFields: data => {
+        return {
+          id: data.id,
+        }
+      },
+    })
+    expect(hasProperty(result, 'author')).toBe(true)
+    expect(hasProperty(result.entries[0], 'id')).toBe(true)
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+
+  test('extract rdf feed from Slashdot with extraFields', async () => {
+    const url = 'https://some-news-page.tld/atom'
+    const xml = readFileSync('test-data/rdf-standard.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      getExtraFeedFields: data => {
+        return {
+          subject: data['dc:subject'],
+        }
+      },
+      getExtraEntryFields: data => {
+        return {
+          author: data['dc:creator'],
+        }
+      },
+    })
+    expect(hasProperty(result, 'subject')).toBe(true)
+    expect(hasProperty(result.entries[0], 'author')).toBe(true)
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+
+  test('extract atom feed which contains multi links', async () => {
+    const url = 'https://some-news-page.tld/atom/multilinks'
+    const xml = readFileSync('test-data/atom-multilinks.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url)
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+
+  test('extract json feed from Micro.blog', async () => {
+    const url = 'https://some-news-page.tld/json'
+    const json = readFileSync('test-data/json-feed-standard-realworld.json', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, json, {
+      'Content-Type': 'text/json',
+    })
+    const result = await extract(url)
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+
+  test('extract json feed from Micro.blog with extra fields', async () => {
+    const url = 'https://some-news-page.tld/json'
+    const json = readFileSync('test-data/json-feed-standard-realworld.json', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, json, {
+      'Content-Type': 'text/json',
+    })
+    const result = await extract(url, {
+      getExtraFeedFields: data => {
+        return {
+          icon: data.icon,
+        }
+      },
+      getExtraEntryFields: data => {
+        return {
+          id: data.id,
+        }
+      },
+    })
+    expect(hasProperty(result, 'icon')).toBe(true)
+    expect(hasProperty(result.entries[0], 'id')).toBe(true)
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+
+  test('extract rss feed from huggingface.co (no link)', async () => {
+    const url = 'https://huggingface.co/no-link/rss'
+    const xml = readFileSync('test-data/rss-feed-miss-link.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url)
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+
+  test('extract rss feed from medium.com (content:encoded)', async () => {
+    const url = 'https://medium.com/feed/@ameliakusiak'
+    const xml = readFileSync('test-data/medium-feed.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url)
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+})
+
+describe('test extract() with `useISODateFormat` option', () => {
+  test('set `useISODateFormat` to false', async () => {
+    const url = 'https://realworld-standard-feed.tld/rss'
+    const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      useISODateFormat: false,
+    })
+    expect(result.published).toEqual('Thu, 28 Jul 2022 03:39:57 GMT')
+    expect(result.entries[0].published).toEqual('Thu, 28 Jul 2022 02:43:00 GMT')
+  })
+
+  test('set `useISODateFormat` to true', async () => {
+    const url = 'https://realworld-standard-feed.tld/rss'
+    const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      useISODateFormat: true,
+    })
+    expect(result.published).toEqual('2022-07-28T03:39:57.000Z')
+    expect(result.entries[0].published).toEqual('2022-07-28T02:43:00.000Z')
+  })
+})
+
+describe('test extract() without normalization', () => {
+  test('extract rss feed from Google', async () => {
+    const url = 'https://some-news-page.tld/rss'
+    const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      normalization: false,
+    })
+    expect(hasProperty(result, 'webMaster')).toBe(true)
+    expect(hasProperty(result, 'item')).toBe(true)
+    expect(hasProperty(result.item[0], 'source')).toBe(true)
+  })
+  test('extract rss feed from standard example', async () => {
+    const url = 'https://some-news-page.tld/rss'
+    const xml = readFileSync('test-data/rss-feed-standard.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      normalization: false,
+    })
+    expect(hasProperty(result, 'copyright')).toBe(true)
+    expect(hasProperty(result, 'item')).toBe(true)
+    expect(hasProperty(result.item, 'guid')).toBe(true)
+  })
+
+  test('extract rdf feed from Slashdot without normalization', async () => {
+    const url = 'https://some-news-page.tld/atom'
+    const xml = readFileSync('test-data/rdf-standard.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      normalization: false,
+    })
+    expect(hasProperty(result.channel, 'syn:updateBase')).toBe(true)
+    expect(hasProperty(result.channel, 'dc:rights')).toBe(true)
+    expect(hasProperty(result, 'item')).toBe(true)
+    expect(hasProperty(result.item[0], 'slash:department')).toBe(true)
+  })
+
+  test('extract atom feed from Google', async () => {
+    const url = 'https://some-news-page.tld/atom'
+    const xml = readFileSync('test-data/atom-feed-standard-realworld.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      normalization: false,
+    })
+    expect(hasProperty(result, 'id')).toBe(true)
+    expect(hasProperty(result, 'rights')).toBe(true)
+    expect(hasProperty(result, 'entry')).toBe(true)
+    expect(hasProperty(result.entry[0], 'updated')).toBe(true)
+  })
+
+  test('extract atom feed from standard example', async () => {
+    const url = 'https://some-news-page.tld/atom'
+    const xml = readFileSync('test-data/atom-feed-standard.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      normalization: false,
+    })
+    expect(hasProperty(result, 'id')).toBe(true)
+    expect(hasProperty(result, 'entry')).toBe(true)
+    expect(hasProperty(result.entry, 'published')).toBe(true)
+    expect(hasProperty(result.entry, 'updated')).toBe(true)
+    expect(hasProperty(result.entry, 'summary')).toBe(true)
+    expect(hasProperty(result.entry, 'content')).toBe(true)
+  })
+
+  test('extract json feed from Micro.blog', async () => {
+    const url = 'https://some-news-page.tld/json'
+    const json = readFileSync('test-data/json-feed-standard-realworld.json', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, json, {
+      'Content-Type': 'application/json',
+    })
+    const result = await extract(url, {
+      normalization: false,
+    })
+    expect(hasProperty(result, 'icon')).toBe(true)
+    expect(hasProperty(result, 'favicon')).toBe(true)
+    expect(hasProperty(result, 'items')).toBe(true)
+    expect(hasProperty(result.items[0], 'tags')).toBe(true)
+    expect(hasProperty(result.items[0], 'date_published')).toBe(true)
+  })
+
+  test('extract rss podcast feed with enclosure tag', async () => {
+    const url = 'https://some-podcast-page.tld/podcast/rss'
+    const xml = readFileSync('test-data/podcast.rss', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, {
+      normalization: false,
+    })
+    expect(hasProperty(result, 'itunes:owner')).toBe(true)
+    expect(hasProperty(result.item[0], 'itunes:duration')).toBe(true)
+  })
+})
+
+describe('test extract with `baseUrl` option', () => {
+  test('extract rss feed from file', () => {
+    const baseUrl = 'https://huggingface.co'
+    const xml = readFileSync('test-data/rss-feed-miss-base-url.xml', 'utf8')
+    const result = extractFromXml(xml, { baseUrl })
+
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+
+    expect(validateProps(result.entries[0])).toBe(true)
+    expect(result.link).toBe(baseUrl + '/blog')
+    expect(result.entries[0].link).toBe(baseUrl + '/blog/intro-graphml')
+  })
+
+  test('extract rdf feed from file', () => {
+    const baseUrl = 'https://slashdot.org'
+    const xml = readFileSync('test-data/rdf-standard.xml', 'utf8')
+    const result = extractFromXml(xml, { baseUrl })
+
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+
+    expect(validateProps(result.entries[0])).toBe(true)
+    expect(result.link).toBe(baseUrl + '/')
+    const firstItemLink = result.entries[0].link
+    expect(firstItemLink.startsWith('https://tech.slashdot.org/story/23/08/23/2238246/spacex-')).toBe(true)
+  })
+
+  test('extract json feed from file', () => {
+    const baseUrl = 'https://www.jsonfeed.org'
+    const json = readFileSync('test-data/json-feed-miss-base-url.json', 'utf8')
+    const result = extractFromJson(JSON.parse(json), { baseUrl })
+
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+
+    expect(result.link).toBe(baseUrl + '/')
+    expect(result.entries[0].link).toBe(baseUrl + '/2020/08/07/json-feed-version.html')
+  })
+
+  test('extract rss feed with url', async () => {
+    const url = 'https://huggingface.co/blog/rss'
+    const xml = readFileSync('test-data/rss-feed-miss-base-url.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, { baseUrl })
+
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+
+    expect(validateProps(result.entries[0])).toBe(true)
+    expect(result.link).toBe(baseUrl + '/blog')
+    expect(result.entries[0].link).toBe(baseUrl + '/blog/intro-graphml')
+  })
+})
+
+if (PROXY_SERVER !== '') {
+  describe('test extract live RSS via proxy server', () => {
+    test('check if extract method works with proxy server', async () => {
+      const url = 'https://news.google.com/rss'
+      const result = await extract(url, {}, {
+        agent: new HttpsProxyAgent(PROXY_SERVER),
+      })
+      expect(result.title).toContain('Google News')
+      expect(result.entries.length).toBeGreaterThan(0)
+    }, 10000)
+  })
+}
+
+describe('check old method read()', () => {
+  test('ensure that depricated method read() still works', async () => {
+    const url = 'https://realworld-standard-feed.tld/rss'
+    const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await read(url, {
+      useISODateFormat: true,
+    })
+    expect(result.published).toEqual('2022-07-28T03:39:57.000Z')
+    expect(result.entries[0].published).toEqual('2022-07-28T02:43:00.000Z')
+  })
+})

node_modules/@extractus/feed-extractor/src/utils/linker.js

@@ -0,0 +1,93 @@
+// utils -> linker
+
+export const isValid = (url = '') => {
+  try {
+    const ourl = new URL(url)
+    return ourl !== null && ourl.protocol.startsWith('http')
+  } catch (err) {
+    return false
+  }
+}
+
+export const absolutify = (fullUrl = '', relativeUrl = '') => {
+  try {
+    const result = new URL(relativeUrl, fullUrl)
+    return result.toString()
+  } catch (err) {
+    return ''
+  }
+}
+
+const blacklistKeys = [
+  'CNDID',
+  '__twitter_impression',
+  '_hsenc',
+  '_openstat',
+  'action_object_map',
+  'action_ref_map',
+  'action_type_map',
+  'amp',
+  'fb_action_ids',
+  'fb_action_types',
+  'fb_ref',
+  'fb_source',
+  'fbclid',
+  'ga_campaign',
+  'ga_content',
+  'ga_medium',
+  'ga_place',
+  'ga_source',
+  'ga_term',
+  'gs_l',
+  'hmb_campaign',
+  'hmb_medium',
+  'hmb_source',
+  'mbid',
+  'mc_cid',
+  'mc_eid',
+  'mkt_tok',
+  'referrer',
+  'spJobID',
+  'spMailingID',
+  'spReportId',
+  'spUserID',
+  'utm_brand',
+  'utm_campaign',
+  'utm_cid',
+  'utm_content',
+  'utm_int',
+  'utm_mailing',
+  'utm_medium',
+  'utm_name',
+  'utm_place',
+  'utm_pubreferrer',
+  'utm_reader',
+  'utm_social',
+  'utm_source',
+  'utm_swu',
+  'utm_term',
+  'utm_userid',
+  'utm_viz_id',
+  'wt_mc_o',
+  'yclid',
+  'WT.mc_id',
+  'WT.mc_ev',
+  'WT.srch',
+  'pk_source',
+  'pk_medium',
+  'pk_campaign',
+]
+
+export const purify = (url) => {
+  try {
+    const pureUrl = new URL(url)
+
+    blacklistKeys.forEach((key) => {
+      pureUrl.searchParams.delete(key)
+    })
+
+    return pureUrl.toString().replace(pureUrl.hash, '')
+  } catch (err) {
+    return null
+  }
+}

node_modules/@extractus/feed-extractor/src/utils/linker.test.js

@@ -0,0 +1,138 @@
+// linker.test
+/* eslint-env jest */
+
+import { isValid, absolutify, purify } from './linker.js'
+
+describe('test exported methods from `linker`', () => {
+  const cases = [
+    {
+      url: 'https://www.23hq.com',
+      expected: true,
+    },
+    {
+      url: 'https://secure.actblue.com',
+      expected: true,
+    },
+    {
+      url: 'https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06',
+      expected: true,
+    },
+    {
+      url: 'http://192.168.1.199:8081/example/page',
+      expected: true,
+    },
+    {
+      url: 'ftp://192.168.1.199:8081/example/page',
+      expected: false,
+    },
+    {
+      url: '',
+      expected: false,
+    },
+    {
+      url: null,
+      expected: false,
+    },
+    {
+      url: { a: 'x' },
+      expected: false,
+    },
+  ]
+  cases.forEach(({ url, expected }) => {
+    test(`isValid("${url}") must return "${expected}"`, () => {
+      const result = isValid(url)
+      expect(result).toEqual(expected)
+    })
+  })
+
+  const entries = [
+    {
+      full: '',
+      expected: '',
+    },
+    {
+      relative: {},
+      expected: '',
+    },
+    {
+      full: 'https://some.where/article/abc-xyz',
+      relative: 'category/page.html',
+      expected: 'https://some.where/article/category/page.html',
+    },
+    {
+      full: 'https://some.where/article/abc-xyz',
+      relative: '../category/page.html',
+      expected: 'https://some.where/category/page.html',
+    },
+    {
+      full: 'https://some.where/blog/authors/article/abc-xyz',
+      relative: '/category/page.html',
+      expected: 'https://some.where/category/page.html',
+    },
+    {
+      full: 'https://some.where/article/abc-xyz',
+      expected: 'https://some.where/article/abc-xyz',
+    },
+  ]
+  entries.forEach((entry) => {
+    const {
+      full,
+      relative,
+      expected,
+    } = entry
+    test(`absolutify("${full}", "${relative}") must become "${expected}"`, () => {
+      const result = absolutify(full, relative)
+      expect(result).toEqual(expected)
+    })
+  })
+
+  test('test url purify() with invalid url', () => {
+    const urls = [
+      null,
+      '',
+      123,
+      {},
+    ]
+    urls.forEach((url) => {
+      const result = purify(url)
+      expect(result).toEqual(null)
+    })
+  })
+
+  test('test url purify() removing regular marketing params', () => {
+    const entries = [
+      {
+        url: 'https://some.where/article/abc-xyz',
+        expected: 'https://some.where/article/abc-xyz',
+      },
+      {
+        url: 'https://some.where/article/abc-xyz#name,bob',
+        expected: 'https://some.where/article/abc-xyz',
+      },
+      {
+        url: 'https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer',
+        expected: 'https://some.where/article/abc-xyz',
+      },
+      {
+        url: 'https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer',
+        expected: 'https://some.where/article/abc-xyz?q=3',
+      },
+      {
+        url: 'https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer',
+        expected: 'https://some.where/article/abc-xyz',
+      },
+      {
+        url: 'https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer',
+        expected: 'https://some.where/article/abc-xyz?q=3',
+      },
+    ]
+    entries.forEach((entry) => {
+      const {
+        url,
+        expected,
+      } = entry
+      const result = purify(url)
+      expect(result).toEqual(expected)
+    })
+  })
+})

node_modules/@extractus/feed-extractor/src/utils/normalizer.js

@@ -0,0 +1,113 @@
+// normalizer
+
+import {
+  isString,
+  isObject,
+  isArray,
+  hasProperty,
+  stripTags,
+  truncate
+} from 'bellajs'
+
+import { decode } from 'html-entities'
+
+import { absolutify, isValid as isValidUrl, purify as purifyUrl } from './linker.js'
+
+export const toISODateString = (dstr) => {
+  try {
+    return dstr ? (new Date(dstr)).toISOString() : ''
+  } catch (err) {
+    return ''
+  }
+}
+
+export const buildDescription = (val, maxlen = 0) => {
+  const stripped = stripTags(String(val).trim().replace(/^<!\[CDATA\[|\]\]>$/g, ''))
+  const text = maxlen > 0 ? truncate(stripped, maxlen) : stripped
+  return text.replace(/\n+/g, ' ')
+}
+
+export const getText = (val) => {
+  const txt = isObject(val) ? (val._text || val['#text'] || val._cdata || val.$t) : val
+  return txt ? decode(String(txt).trim()) : ''
+}
+
+export const getLink = (val = [], id = '') => {
+  if (isObject(id) && hasProperty(id, '@_isPermaLink') && id['@_isPermaLink'] === 'true') {
+    return getText(id)
+  }
+  const getEntryLink = (links) => {
+    const items = links.map((item) => {
+      return getLink(item)
+    })
+    return items.length > 0 ? items[0] : ''
+  }
+  const url = isString(val)
+    ? getText(val)
+    : isObject(val) && hasProperty(val, 'href')
+      ? getText(val.href)
+      : isObject(val) && hasProperty(val, '@_href')
+        ? getText(val['@_href'])
+        : isObject(val) && hasProperty(val, '@_url')
+          ? getText(val['@_url'])
+          : isObject(val) && hasProperty(val, '_attributes')
+            ? getText(val._attributes.href)
+            : isArray(val) ? getEntryLink(val) : ''
+
+  return url ? url : isValidUrl(id) ? id : ''
+}
+
+export const getPureUrl = (url, id = '', baseUrl) => {
+  const link = getLink(url, id)
+  const pu = purifyUrl(link)
+
+  return link
+    ? pu
+      ? pu
+      : absolutify(baseUrl, link)
+    : ''
+}
+
+const hash = (str) => Math.abs(str.split('').reduce((s, c) => Math.imul(31, s) + c.charCodeAt(0) | 0, 0)).toString(36)
+
+export const getEntryId = (id, url, pubDate) => {
+  return id ? getText(id) : hash(getPureUrl(url)) + '-' + (new Date(pubDate)).getTime()
+}
+
+export const getEnclosure = (val) => {
+  const url = hasProperty(val, '@_url') ? val['@_url'] : ''
+  const type = hasProperty(val, '@_type') ? val['@_type'] : ''
+  const length = Number(hasProperty(val, '@_length') ? val['@_length'] : 0)
+  return !url || !type
+    ? null
+    : {
+      url,
+      type,
+      length,
+    }
+}
+
+const getCategory = (v) => {
+  return isObject(v)
+    ? {
+      text: getText(v),
+      domain: v['@_domain'],
+    }
+    : v
+}
+
+export const getOptionalTags = (val, key) => {
+  if (key === 'source') {
+    return {
+      text: getText(val),
+      url: getLink(val),
+    }
+  }
+  if (key === 'category') {
+    return isArray(val) ? val.map(getCategory) : getCategory(val)
+  }
+  if (key === 'enclosure') {
+    return getEnclosure(val)
+  }
+  return val
+}

node_modules/@extractus/feed-extractor/src/utils/normalizer.test.js

@@ -0,0 +1,13 @@
+// normalizer.test
+/* eslint-env jest */
+
+import { toISODateString } from './normalizer.js'
+
+describe('test `normalizer` methods', () => {
+  test('test toISODateString()', () => {
+    expect(toISODateString('Thu, 28 Jul 2022 08:59:58 GMT')).toEqual('2022-07-28T08:59:58.000Z')
+    expect(toISODateString('2022-07-28T02:43:00.000000000Z')).toEqual('2022-07-28T02:43:00.000Z')
+    expect(toISODateString('')).toEqual('')
+    expect(toISODateString('Thi, 280 Jul 2022 108:79:68 XMT')).toEqual('')
+  })
+})

node_modules/@extractus/feed-extractor/src/utils/parseAtomFeed.js

@@ -0,0 +1,140 @@
+// parseAtomFeed.js
+
+// specs: https://datatracker.ietf.org/doc/html/rfc5023
+// refer: https://validator.w3.org/feed/docs/atom.html
+
+import { isArray, hasProperty } from 'bellajs'
+
+import {
+  getText,
+  toISODateString,
+  buildDescription,
+  getPureUrl,
+  getEntryId
+} from './normalizer.js'
+
+const transform = (item, options) => {
+  const {
+    useISODateFormat,
+    descriptionMaxLen,
+    baseUrl,
+    getExtraEntryFields,
+  } = options
+
+  const {
+    id = '',
+    title = '',
+    issued = '',
+    modified = '',
+    updated = '',
+    published = '',
+    link = '',
+    summary = '',
+    content = '',
+  } = item
+
+  const pubDate = updated || modified || published || issued
+  const htmlContent = getText(summary || content)
+  const entry = {
+    id: getEntryId(id, link, pubDate),
+    title: getText(title),
+    link: getPureUrl(link, id, baseUrl),
+    published: useISODateFormat ? toISODateString(pubDate) : pubDate,
+    description: buildDescription(htmlContent, descriptionMaxLen),
+  }
+
+  const extraFields = getExtraEntryFields(item)
+
+  return {
+    ...entry,
+    ...extraFields,
+  }
+}
+
+const flatten = (feed, baseUrl) => {
+  const {
+    id,
+    title = '',
+    link = '',
+    entry,
+  } = feed
+
+  const entries = isArray(entry) ? entry : [entry]
+  const items = entries.map((entry) => {
+    const {
+      id,
+      title = '',
+      link = '',
+      summary = '',
+      content = '',
+    } = entry
+    const item = {
+      ...entry,
+      title: getText(title),
+      link: getPureUrl(link, id, baseUrl),
+    }
+    if (hasProperty(item, 'summary')) {
+      item.summary = getText(summary)
+    }
+    if (hasProperty(item, 'content')) {
+      item.content = getText(content)
+    }
+    return item
+  })
+
+  const output = {
+    ...feed,
+    title: getText(title),
+    link: getPureUrl(link, id, baseUrl),
+    entry: isArray(entry) ? items : items[0],
+  }
+  return output
+}
+
+const parseAtom = (data, options = {}) => {
+  const {
+    normalization,
+    baseUrl,
+    getExtraFeedFields,
+  } = options
+
+  const feedData = data.feed
+
+  if (!normalization) {
+    return flatten(feedData, baseUrl)
+  }
+
+  const {
+    id = '',
+    title = '',
+    link = '',
+    subtitle = '',
+    generator = '',
+    language = '',
+    updated = '',
+    entry: item = [],
+  } = feedData
+
+  const extraFields = getExtraFeedFields(feedData)
+
+  const items = isArray(item) ? item : [item]
+
+  const published = options.useISODateFormat ? toISODateString(updated) : updated
+
+  return {
+    title: getText(title),
+    link: getPureUrl(link, id, baseUrl),
+    description: subtitle,
+    language,
+    generator,
+    published,
+    ...extraFields,
+    entries: items.map((item) => {
+      return transform(item, options)
+    }),
+  }
+}
+
+export default (data, options = {}) => {
+  return parseAtom(data, options)
+}

node_modules/@extractus/feed-extractor/src/utils/parseJsonFeed.js

@@ -0,0 +1,89 @@
+// parseJsonFeed.js
+
+// specs: https://www.jsonfeed.org/version/1.1/
+
+import { isArray } from 'bellajs'
+
+import {
+  toISODateString,
+  buildDescription,
+  getEntryId
+} from './normalizer.js'
+
+import { absolutify, purify as purifyUrl } from './linker.js'
+
+const transform = (item, options) => {
+  const {
+    useISODateFormat,
+    descriptionMaxLen,
+    baseUrl,
+    getExtraEntryFields,
+  } = options
+
+  const {
+    id = '',
+    title = '',
+    url: link = '',
+    date_published: pubDate = '',
+    summary = '',
+    content_html: htmlContent = '',
+    content_text: textContent = '',
+  } = item
+
+  const published = useISODateFormat ? toISODateString(pubDate) : pubDate
+  const extraFields = getExtraEntryFields(item)
+
+  const entry = {
+    id: getEntryId(id, link, pubDate),
+    title,
+    link: purifyUrl(link) || absolutify(baseUrl, link),
+    published,
+    description: buildDescription(textContent || htmlContent || summary, descriptionMaxLen),
+  }
+
+  return {
+    ...entry,
+    ...extraFields,
+  }
+}
+
+const parseJson = (data, options) => {
+  const {
+    normalization,
+    baseUrl,
+    getExtraFeedFields,
+  } = options
+
+  if (!normalization) {
+    return data
+  }
+
+  const {
+    title = '',
+    home_page_url: homepageUrl = '',
+    description = '',
+    language = '',
+    items: item = [],
+  } = data
+
+  const extraFields = getExtraFeedFields(data)
+
+  const items = isArray(item) ? item : [item]
+
+  return {
+    title,
+    link: purifyUrl(homepageUrl) || absolutify(baseUrl, homepageUrl),
+    description,
+    language,
+    published: '',
+    generator: '',
+    ...extraFields,
+    entries: items.map((item) => {
+      return transform(item, options)
+    }),
+  }
+}
+
+export default (data, options = {}) => {
+  return parseJson(data, options)
+}

node_modules/@extractus/feed-extractor/src/utils/parseRdfFeed.js

@@ -0,0 +1,129 @@
+// parseRssFeed.js
+
+// specs: https://www.rssboard.org/rss-specification
+
+import { isArray } from 'bellajs'
+
+import {
+  getText,
+  toISODateString,
+  buildDescription,
+  getPureUrl,
+  getEntryId
+} from './normalizer.js'
+
+const transform = (item, options) => {
+  const {
+    useISODateFormat,
+    descriptionMaxLen,
+    baseUrl,
+    getExtraEntryFields,
+  } = options
+
+  const {
+    guid = '',
+    title = '',
+    link = '',
+    'dc:date': pubDate = '',
+    description = '',
+    'content:encoded': content = '',
+  } = item
+
+  const published = useISODateFormat ? toISODateString(pubDate) : pubDate
+  const htmlContent = getText(description || content)
+  const entry = {
+    id: getEntryId(guid, link, pubDate),
+    title: getText(title),
+    link: getPureUrl(link, guid, baseUrl),
+    published,
+    description: buildDescription(htmlContent, descriptionMaxLen),
+  }
+
+  const extraFields = getExtraEntryFields(item)
+
+  return {
+    ...entry,
+    ...extraFields,
+  }
+}
+
+const flatten = (feed, baseUrl) => {
+  const {
+    title = '',
+    link = '',
+    item,
+  } = feed
+
+  const items = isArray(item) ? item : [item]
+  const entries = items.map((entry) => {
+    const {
+      id,
+      title = '',
+      link = '',
+    } = entry
+
+    const item = {
+      ...entry,
+      title: getText(title),
+      link: getPureUrl(link, id, baseUrl),
+    }
+
+    return item
+  })
+
+  const output = {
+    ...feed,
+    title: getText(title),
+    link: getPureUrl(link, baseUrl),
+    item: isArray(item) ? entries : entries[0],
+  }
+  return output
+}
+
+const parseRdf = (data, options = {}) => {
+  const {
+    normalization,
+    baseUrl,
+    getExtraFeedFields,
+  } = options
+
+  const feedData = data['rdf:RDF']
+
+  if (!normalization) {
+    return flatten(feedData, baseUrl)
+  }
+
+  const {
+    title = '',
+    link = '',
+    description = '',
+    generator = '',
+    'dc:language': language = '',
+    'dc:date': lastBuildDate = '',
+  } = feedData.channel
+
+  const { item } = feedData
+
+  const extraFields = getExtraFeedFields(feedData)
+
+  const items = isArray(item) ? item : [item]
+
+  const published = options.useISODateFormat ? toISODateString(lastBuildDate) : lastBuildDate
+
+  return {
+    title: getText(title),
+    link: getPureUrl(link, '', baseUrl),
+    description,
+    language,
+    generator,
+    published,
+    ...extraFields,
+    entries: items.map((item) => {
+      return transform(item, options)
+    }),
+  }
+}
+
+export default (data, options = {}) => {
+  return parseRdf(data, options)
+}

node_modules/@extractus/feed-extractor/src/utils/parseRssFeed.js

@@ -0,0 +1,144 @@
+// parseRssFeed.js
+
+// specs: https://www.rssboard.org/rss-specification
+
+import { isArray, hasProperty } from 'bellajs'
+
+import {
+  getText,
+  toISODateString,
+  buildDescription,
+  getPureUrl,
+  getOptionalTags,
+  getEntryId
+} from './normalizer.js'
+
+const transform = (item, options) => {
+  const {
+    useISODateFormat,
+    descriptionMaxLen,
+    baseUrl,
+    getExtraEntryFields,
+  } = options
+
+  const {
+    guid = '',
+    title = '',
+    link = '',
+    pubDate = '',
+    description = '',
+    'content:encoded': content = '',
+  } = item
+
+  const published = useISODateFormat ? toISODateString(pubDate) : pubDate
+  const htmlContent = getText(description || content)
+  const entry = {
+    id: getEntryId(guid, link, pubDate),
+    title: getText(title),
+    link: getPureUrl(link, guid, baseUrl),
+    published,
+    description: buildDescription(htmlContent, descriptionMaxLen),
+  }
+
+  const extraFields = getExtraEntryFields(item)
+
+  return {
+    ...entry,
+    ...extraFields,
+  }
+}
+
+const flatten = (feed, baseUrl) => {
+  const {
+    title = '',
+    link = '',
+    item,
+  } = feed
+
+  const items = isArray(item) ? item : [item]
+  const entries = items.map((entry) => {
+    const {
+      id,
+      title = '',
+      link = '',
+    } = entry
+
+    const item = {
+      ...entry,
+      title: getText(title),
+      link: getPureUrl(link, id, baseUrl),
+    }
+
+    const txtTags = 'guid description source'.split(' ')
+
+    txtTags.forEach((key) => {
+      if (hasProperty(entry, key)) {
+        item[key] = getText(entry[key])
+      }
+    })
+
+    const optionalProps = 'source category enclosure author image'.split(' ')
+    optionalProps.forEach((key) => {
+      if (hasProperty(item, key)) {
+        entry[key] = getOptionalTags(item[key], key)
+      }
+    })
+
+    return item
+  })
+
+  const output = {
+    ...feed,
+    title: getText(title),
+    link: getPureUrl(link, baseUrl),
+    item: isArray(item) ? entries : entries[0],
+  }
+  return output
+}
+
+const parseRss = (data, options = {}) => {
+  const {
+    normalization,
+    baseUrl,
+    getExtraFeedFields,
+  } = options
+
+  const feedData = data.rss.channel
+
+  if (!normalization) {
+    return flatten(feedData, baseUrl)
+  }
+
+  const {
+    title = '',
+    link = '',
+    description = '',
+    generator = '',
+    language = '',
+    lastBuildDate = '',
+    item = [],
+  } = feedData
+
+  const extraFields = getExtraFeedFields(feedData)
+
+  const items = isArray(item) ? item : [item]
+
+  const published = options.useISODateFormat ? toISODateString(lastBuildDate) : lastBuildDate
+
+  return {
+    title: getText(title),
+    link: getPureUrl(link, '', baseUrl),
+    description,
+    language,
+    generator,
+    published,
+    ...extraFields,
+    entries: items.map((item) => {
+      return transform(item, options)
+    }),
+  }
+}
+
+export default (data, options = {}) => {
+  return parseRss(data, options)
+}

node_modules/@extractus/feed-extractor/src/utils/retrieve.js

@@ -0,0 +1,50 @@
+// utils -> retrieve
+
+import fetch from 'cross-fetch'
+
+const profetch = async (url, options = {}) => {
+  const { proxy = {}, signal = null } = options
+  const {
+    target,
+    headers = {},
+  } = proxy
+  const res = await fetch(target + encodeURIComponent(url), {
+    headers,
+    signal,
+  })
+  return res
+}
+
+export default async (url, options = {}) => {
+  const {
+    headers = {
+      'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0',
+    },
+    proxy = null,
+    agent = null,
+    signal = null,
+  } = options
+
+  const res = proxy ? await profetch(url, { proxy, signal }) : await fetch(url, { headers, agent, signal })
+
+  const status = res.status
+  if (status >= 400) {
+    throw new Error(`Request failed with error code ${status}`)
+  }
+  const contentType = res.headers.get('content-type')
+  const text = await res.text()
+
+  if (/(\+|\/)(xml|html)/.test(contentType)) {
+    return { type: 'xml', text: text.trim(), status, contentType }
+  }
+
+  if (/(\+|\/)json/.test(contentType)) {
+    try {
+      const data = JSON.parse(text)
+      return { type: 'json', json: data, status, contentType }
+    } catch (err) {
+      throw new Error('Failed to convert data to JSON object')
+    }
+  }
+  throw new Error(`Invalid content type: ${contentType}`)
+}

node_modules/@extractus/feed-extractor/src/utils/retrieve.test.js

@@ -0,0 +1,76 @@
+// retrieve.test
+/* eslint-env jest */
+
+import nock from 'nock'
+
+import retrieve from './retrieve.js'
+
+const parseUrl = (url) => {
+  const re = new URL(url)
+  return {
+    baseUrl: `${re.protocol}//${re.host}`,
+    path: re.pathname,
+  }
+}
+
+describe('test retrieve() method', () => {
+  test('test retrieve with bad status code', async () => {
+    const url = 'https://some.where/bad/page'
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(500, 'Error 500')
+    expect(retrieve(url)).rejects.toThrow(new Error('Request failed with error code 500'))
+  })
+
+  test('test retrieve with bad conten type', async () => {
+    const url = 'https://some.where/bad/page'
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, '<?xml version="1.0"?><tag>this is xml</tag>', {
+      'Content-Type': 'something/type',
+    })
+    expect(retrieve(url)).rejects.toThrow(new Error('Invalid content type: something/type'))
+  })
+
+  test('test retrieve from good source', async () => {
+    const url = 'https://some.where/good/page'
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, '<div>this is content</div>', {
+      'Content-Type': 'application/rss+xml',
+    })
+    const result = await retrieve(url)
+    expect(result.type).toEqual('xml')
+    expect(result.text).toEqual('<div>this is content</div>')
+  })
+
+  test('test retrieve from good source, but having \\r\\n before/after root xml', async () => {
+    const url = 'https://some.where/good/page'
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, '\n\r\r\n\n<div>this is content</div>\n\r\r\n\n', {
+      'Content-Type': 'text/xml',
+    })
+    const result = await retrieve(url)
+    expect(result.type).toEqual('xml')
+    expect(result.text).toBe('<div>this is content</div>')
+  })
+
+  test('test retrieve using proxy', async () => {
+    const url = 'https://some.where/good/source-with-proxy'
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, 'something bad', {
+      'Content-Type': 'bad/thing',
+    })
+    nock('https://proxy-server.com')
+      .get('/api/proxy?url=https%3A%2F%2Fsome.where%2Fgood%2Fsource-with-proxy')
+      .reply(200, '<?xml version="1.0"?><tag>this is xml</tag>', {
+        'Content-Type': 'text/xml',
+      })
+
+    const result = await retrieve(url, {
+      proxy: {
+        target: 'https://proxy-server.com/api/proxy?url=',
+      },
+    })
+    expect(result.type).toEqual('xml')
+    expect(result.text).toEqual('<?xml version="1.0"?><tag>this is xml</tag>')
+    nock.cleanAll()
+  })
+})

node_modules/@extractus/feed-extractor/src/utils/xmlparser.js

@@ -0,0 +1,32 @@
+// utils / xmlparser
+
+import { hasProperty, isString } from 'bellajs'
+
+import { XMLValidator, XMLParser } from 'fast-xml-parser'
+
+export const isRSS = (data = {}) => {
+  return hasProperty(data, 'rss') && hasProperty(data.rss, 'channel')
+}
+
+export const isAtom = (data = {}) => {
+  return hasProperty(data, 'feed') && hasProperty(data.feed, 'entry')
+}
+
+export const isRdf = (data = {}) => {
+  return hasProperty(data, 'rdf:RDF') && hasProperty(data['rdf:RDF'], 'channel')
+}
+
+export const validate = (xml) => {
+  return (!isString(xml) || !xml.length) ? false : XMLValidator.validate(xml) === true
+}
+
+export const xml2obj = (xml = '', extraOptions = {}) => {
+  const options = {
+    attributeNamePrefix: '@_',
+    ignoreAttributes: false,
+    ...extraOptions,
+  }
+  const parser = new XMLParser(options)
+  const jsonObj = parser.parse(xml)
+  return jsonObj
+}

node_modules/@extractus/feed-extractor/src/utils/xmlparser.test.js

@@ -0,0 +1,41 @@
+// xmlparser.test
+/* eslint-env jest */
+
+import { readFileSync } from 'fs'
+
+import { validate, isRSS, isAtom, xml2obj } from './xmlparser.js'
+
+describe('test methods from `xmlparser`', () => {
+  test('test validate(well format xml)', async () => {
+    const xmlData = '<xml><atag id="12">value</atag></xml>'
+    const result = validate(xmlData)
+    expect(result).toBe(true)
+  })
+
+  test('test validate(bad format xml)', async () => {
+    const xmlData = '<xml><atag id="12">value</btag></xml>'
+    const result = validate(xmlData)
+    expect(result).toBe(false)
+  })
+
+  test('test validate(standard rss content)', async () => {
+    const xml = readFileSync('test-data/rss-feed-standard.xml', 'utf8')
+    const xmlData = xml2obj(xml)
+    expect(isRSS(xmlData)).toBe(true)
+    expect(isAtom(xmlData)).toBe(false)
+  })
+
+  test('test validate(standard atom content)', async () => {
+    const xml = readFileSync('test-data/atom-feed-standard.xml', 'utf8')
+    const xmlData = xml2obj(xml)
+    expect(isAtom(xmlData)).toBe(true)
+    expect(isRSS(xmlData)).toBe(false)
+  })
+
+  test('test xml2obj(well format xml)', async () => {
+    const xmlData = '<xml><atag id="12">value</atag></xml>'
+    const result = xml2obj(xmlData)
+    expect(result).toBeInstanceOf(Object)
+    expect(result.xml).toBeInstanceOf(Object)
+  })
+})