remove node_modules and .gitignore them
This commit is contained in:
-2
@@ -1,2 +0,0 @@
|
||||
node_modules
|
||||
dist
|
||||
-121
@@ -1,121 +0,0 @@
|
||||
{
|
||||
"parserOptions": {
|
||||
"ecmaVersion": "latest",
|
||||
"sourceType": "module"
|
||||
},
|
||||
"env": {
|
||||
"es6": true,
|
||||
"node": true,
|
||||
"browser": true,
|
||||
"jest": true
|
||||
},
|
||||
"globals": {
|
||||
"globalThis": true
|
||||
},
|
||||
"plugins": [],
|
||||
"overrides": [],
|
||||
"extends": ["eslint:recommended"],
|
||||
"rules": {
|
||||
"arrow-spacing": ["error", { "before": true, "after": true }],
|
||||
"block-spacing": ["error", "always"],
|
||||
"brace-style": ["error", "1tbs", { "allowSingleLine": true }],
|
||||
"camelcase": ["error", {
|
||||
"allow": ["^UNSAFE_"],
|
||||
"properties": "never",
|
||||
"ignoreGlobals": true
|
||||
}],
|
||||
"comma-dangle": ["error", {
|
||||
"arrays": "always-multiline",
|
||||
"objects": "always-multiline",
|
||||
"imports": "never",
|
||||
"exports": "never",
|
||||
"functions": "never"
|
||||
}],
|
||||
"comma-spacing": ["error", { "before": false, "after": true }],
|
||||
"eol-last": "error",
|
||||
"eqeqeq": ["error", "always", { "null": "ignore" }],
|
||||
"func-call-spacing": ["error", "never"],
|
||||
"indent": [
|
||||
"error",
|
||||
2,
|
||||
{
|
||||
"MemberExpression": 1,
|
||||
"FunctionDeclaration": {
|
||||
"body": 1,
|
||||
"parameters": 2
|
||||
},
|
||||
"SwitchCase": 1
|
||||
}
|
||||
],
|
||||
"key-spacing": ["error", { "beforeColon": false, "afterColon": true }],
|
||||
"keyword-spacing": ["error", { "before": true, "after": true }],
|
||||
"lines-between-class-members": ["error", "always", { "exceptAfterSingleLine": true }],
|
||||
"max-len": [
|
||||
"error",
|
||||
{
|
||||
"code": 120,
|
||||
"ignoreTrailingComments": true,
|
||||
"ignoreComments": true,
|
||||
"ignoreUrls": true
|
||||
}
|
||||
],
|
||||
"max-lines": [
|
||||
"error",
|
||||
{
|
||||
"max": 520,
|
||||
"skipBlankLines": true,
|
||||
"skipComments": false
|
||||
}
|
||||
],
|
||||
"max-lines-per-function": [
|
||||
"error",
|
||||
{
|
||||
"max": 240,
|
||||
"skipBlankLines": true
|
||||
}
|
||||
],
|
||||
"max-params": ["error", 3],
|
||||
"no-array-constructor": "error",
|
||||
"no-mixed-spaces-and-tabs": "error",
|
||||
"no-multi-spaces": "error",
|
||||
"no-multi-str": "error",
|
||||
"no-multiple-empty-lines": [
|
||||
"error",
|
||||
{
|
||||
"max": 1,
|
||||
"maxEOF": 0
|
||||
}
|
||||
],
|
||||
"no-restricted-syntax": [
|
||||
"error",
|
||||
"WithStatement",
|
||||
"BinaryExpression[operator='in']"
|
||||
],
|
||||
"no-trailing-spaces": "error",
|
||||
"no-use-before-define": [
|
||||
"error",
|
||||
{
|
||||
"functions": true,
|
||||
"classes": true,
|
||||
"variables": false
|
||||
}
|
||||
],
|
||||
"no-var": "warn",
|
||||
"object-curly-spacing": ["error", "always"],
|
||||
"padded-blocks": [
|
||||
"error",
|
||||
{
|
||||
"blocks": "never",
|
||||
"switches": "never",
|
||||
"classes": "never"
|
||||
}
|
||||
],
|
||||
"quotes": ["error", "single"],
|
||||
"space-before-blocks": ["error", "always"],
|
||||
"space-before-function-paren": ["error", "always"],
|
||||
"space-infix-ops": "error",
|
||||
"space-unary-ops": ["error", { "words": true, "nonwords": false }],
|
||||
"space-in-parens": ["error", "never"],
|
||||
"semi": ["error", "never"]
|
||||
}
|
||||
}
|
||||
-71
@@ -1,71 +0,0 @@
|
||||
# Contributing to `@extractus/feed-extractor`
|
||||
|
||||
Glad to see you here.
|
||||
|
||||
Collaborations and pull requests are always welcomed, though larger proposals should be discussed first.
|
||||
|
||||
As an OSS, it's better to follow the Unix philosophy: "do one thing and do it well".
|
||||
|
||||
|
||||
## Third-party libraries
|
||||
|
||||
Please avoid using libaries other than those available in the standard library, unless necessary.
|
||||
|
||||
This library needs to be simple and flexible to run on multiple platforms such as Deno, Bun, or even browser.
|
||||
|
||||
|
||||
## Coding convention
|
||||
|
||||
Make sure your code lints before opening a pull request.
|
||||
|
||||
```bash
|
||||
cd feed-extractor
|
||||
|
||||
# check coding convention issue
|
||||
npm run lint
|
||||
|
||||
# auto fix coding convention issue
|
||||
npm run lint:fix
|
||||
```
|
||||
|
||||
*When you run `npm test`, the linting process will be triggered at first.*
|
||||
|
||||
|
||||
## Testing
|
||||
|
||||
Be sure to run the unit test suite before opening a pull request. An example test run is shown below.
|
||||
|
||||
```bash
|
||||
cd feed-extractor
|
||||
npm test
|
||||
```
|
||||
|
||||
|
||||
|
||||
If test coverage decreased, please check test scripts and try to improve this number.
|
||||
|
||||
|
||||
## Documentation
|
||||
|
||||
If you've changed APIs, please update README and [the examples](examples).
|
||||
|
||||
|
||||
## Clean commit histories
|
||||
|
||||
When you open a pull request, please ensure the commit history is clean.
|
||||
Squash the commits into logical blocks, perhaps a single commit if that makes sense.
|
||||
|
||||
What you want to avoid is commits such as "WIP" and "fix test" in the history.
|
||||
This is so we keep history on master clean and straightforward.
|
||||
|
||||
For people new to git, please refer the following guides:
|
||||
|
||||
- [Writing good commit messages](https://github.com/erlang/otp/wiki/writing-good-commit-messages)
|
||||
- [Commit Message Guidelines](https://gist.github.com/robertpainsi/b632364184e70900af4ab688decf6f53)
|
||||
|
||||
|
||||
## License
|
||||
|
||||
By contributing to `@extractus/feed-extractor`, you agree that your contributions will be licensed under its [MIT license](LICENSE).
|
||||
|
||||
---
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015 Extractus
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
-380
@@ -1,380 +0,0 @@
|
||||
# feed-extractor
|
||||
|
||||
To read & normalize RSS/ATOM/JSON feed data.
|
||||
|
||||
[](https://badge.fury.io/js/@extractus%2Ffeed-extractor)
|
||||
|
||||
|
||||
[](https://coveralls.io/github/extractus/feed-extractor?branch=main)
|
||||
|
||||
(This library is derived from [feed-reader](https://www.npmjs.com/package/feed-reader) renamed.)
|
||||
|
||||
## Demo
|
||||
|
||||
- [Give it a try!](https://extractor-demos.pages.dev/feed-extractor)
|
||||
- [Example FaaS](https://extractus.deno.dev/extract?apikey=rn0wbHos2e73W6ghQf705bdF&type=feed&url=https://news.google.com/rss)
|
||||
|
||||
## Install & Usage
|
||||
|
||||
### Node.js
|
||||
|
||||
```bash
|
||||
npm i @extractus/feed-extractor
|
||||
```
|
||||
|
||||
```ts
|
||||
import { extract } from '@extractus/feed-extractor'
|
||||
|
||||
// extract a RSS
|
||||
const result = await extract('https://news.google.com/rss')
|
||||
console.log(result)
|
||||
```
|
||||
|
||||
### Deno
|
||||
|
||||
```ts
|
||||
// deno < 1.28
|
||||
import { extract } from 'https://esm.sh/@extractus/feed-extractor'
|
||||
|
||||
// deno > 1.28
|
||||
import { extract } from 'npm:@extractus/feed-extractor'
|
||||
```
|
||||
|
||||
### Browser
|
||||
|
||||
```ts
|
||||
import { extract } from 'https://esm.sh/@extractus/feed-extractor'
|
||||
```
|
||||
|
||||
Please check [the examples](https://github.com/extractus/feed-extractor/tree/main/examples) for reference.
|
||||
|
||||
|
||||
## APIs
|
||||
|
||||
- [extract()](#extract)
|
||||
- [extractFromJson()](#extractfromjson)
|
||||
- [extractFromXml()](#extractfromxml)
|
||||
|
||||
#### Note:
|
||||
|
||||
- *Old method `read()` has been marked as deprecated and will be removed in next major release.*
|
||||
|
||||
---
|
||||
|
||||
### `extract()`
|
||||
|
||||
Load and extract feed data from given RSS/ATOM/JSON source. Return a Promise object.
|
||||
|
||||
#### Syntax
|
||||
|
||||
```ts
|
||||
extract(String url)
|
||||
extract(String url, Object parserOptions)
|
||||
extract(String url, Object parserOptions, Object fetchOptions)
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
import { extract } from '@extractus/feed-extractor'
|
||||
|
||||
const result = await extract('https://news.google.com/atom')
|
||||
console.log(result)
|
||||
```
|
||||
|
||||
Without any options, the result should have the following structure:
|
||||
|
||||
```ts
|
||||
{
|
||||
title: String,
|
||||
link: String,
|
||||
description: String,
|
||||
generator: String,
|
||||
language: String,
|
||||
published: ISO Date String,
|
||||
entries: Array[
|
||||
{
|
||||
id: String,
|
||||
title: String,
|
||||
link: String,
|
||||
description: String,
|
||||
published: ISO Datetime String
|
||||
},
|
||||
// ...
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
##### `url` *required*
|
||||
|
||||
URL of a valid feed source
|
||||
|
||||
Feed content must be accessible and conform one of the following standards:
|
||||
|
||||
- [RSS Feed](https://www.rssboard.org/rss-specification)
|
||||
- [RDF Feed](https://web.resource.org/rss/1.0/spec)
|
||||
- [ATOM Feed](https://datatracker.ietf.org/doc/html/rfc5023)
|
||||
- [JSON Feed](https://www.jsonfeed.org/version/1.1/)
|
||||
|
||||
##### `parserOptions` *optional*
|
||||
|
||||
Object with all or several of the following properties:
|
||||
|
||||
- `normalization`: Boolean, normalize feed data or keep original. Default `true`.
|
||||
- `useISODateFormat`: Boolean, convert datetime to ISO format. Default `true`.
|
||||
- `descriptionMaxLen`: Number, to truncate description. Default `250` characters. Set to `0` = no truncation.
|
||||
- `xmlParserOptions`: Object, used by xml parser, view [fast-xml-parser's docs](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md)
|
||||
- `getExtraFeedFields`: Function, to get more fields from feed data
|
||||
- `getExtraEntryFields`: Function, to get more fields from feed entry data
|
||||
- `baseUrl`: URL string, to absolutify the links within feed content
|
||||
|
||||
For example:
|
||||
|
||||
```ts
|
||||
import { extract } from '@extractus/feed-extractor'
|
||||
|
||||
await extract('https://news.google.com/atom', {
|
||||
useISODateFormat: false
|
||||
})
|
||||
|
||||
await extract('https://news.google.com/rss', {
|
||||
useISODateFormat: false,
|
||||
getExtraFeedFields: (feedData) => {
|
||||
return {
|
||||
subtitle: feedData.subtitle || ''
|
||||
}
|
||||
},
|
||||
getExtraEntryFields: (feedEntry) => {
|
||||
const {
|
||||
enclosure,
|
||||
category
|
||||
} = feedEntry
|
||||
return {
|
||||
enclosure: {
|
||||
url: enclosure['@_url'],
|
||||
type: enclosure['@_type'],
|
||||
length: enclosure['@_length']
|
||||
},
|
||||
category: isString(category) ? category : {
|
||||
text: category['@_text'],
|
||||
domain: category['@_domain']
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
##### `fetchOptions` *optional*
|
||||
|
||||
`fetchOptions` is an object that can have the following properties:
|
||||
|
||||
- `headers`: to set request headers
|
||||
- `proxy`: another endpoint to forward the request to
|
||||
- `agent`: a HTTP proxy agent
|
||||
- `signal`: AbortController signal or AbortSignal timeout to terminate the request
|
||||
|
||||
For example, you can use this param to set request headers to fetch as below:
|
||||
|
||||
```js
|
||||
import { extract } from '@extractus/feed-extractor'
|
||||
|
||||
const url = 'https://news.google.com/rss'
|
||||
await extract(url, null, {
|
||||
headers: {
|
||||
'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1'
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
You can also specify a proxy endpoint to load remote content, instead of fetching directly.
|
||||
|
||||
For example:
|
||||
|
||||
```js
|
||||
import { extract } from '@extractus/feed-extractor'
|
||||
|
||||
const url = 'https://news.google.com/rss'
|
||||
|
||||
await extract(url, null, {
|
||||
headers: {
|
||||
'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1'
|
||||
},
|
||||
proxy: {
|
||||
target: 'https://your-secret-proxy.io/loadXml?url=',
|
||||
headers: {
|
||||
'Proxy-Authorization': 'Bearer YWxhZGRpbjpvcGVuc2VzYW1l...'
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
Passing requests to proxy is useful while running `@extractus/feed-extractor` on browser.
|
||||
View `examples/browser-feed-reader` as reference example.
|
||||
|
||||
Another way to work with proxy is use `agent` option instead of `proxy` as below:
|
||||
|
||||
```js
|
||||
import { extract } from '@extractus/feed-extractor'
|
||||
|
||||
import { HttpsProxyAgent } from 'https-proxy-agent'
|
||||
|
||||
const proxy = 'http://abc:RaNdoMpasswORd_country-France@proxy.packetstream.io:31113'
|
||||
|
||||
const url = 'https://news.google.com/rss'
|
||||
|
||||
const feed = await extract(url, null, {
|
||||
agent: new HttpsProxyAgent(proxy),
|
||||
})
|
||||
console.log('Run feed-extractor with proxy:', proxy)
|
||||
console.log(feed)
|
||||
```
|
||||
|
||||
For more info about [https-proxy-agent](https://www.npmjs.com/package/https-proxy-agent), check [its repo](https://github.com/TooTallNate/proxy-agents).
|
||||
|
||||
By default, there is no request timeout. You can use the option `signal` to cancel request at the right time.
|
||||
|
||||
The common way is to use AbortControler:
|
||||
|
||||
```js
|
||||
const controller = new AbortController()
|
||||
|
||||
// stop after 5 seconds
|
||||
setTimeout(() => {
|
||||
controller.abort()
|
||||
}, 5000)
|
||||
|
||||
const data = await extract(url, null, {
|
||||
signal: controller.signal,
|
||||
})
|
||||
```
|
||||
|
||||
A newer solution is AbortSignal's `timeout()` static method:
|
||||
|
||||
```js
|
||||
// stop after 5 seconds
|
||||
const data = await extract(url, null, {
|
||||
signal: AbortSignal.timeout(5000),
|
||||
})
|
||||
```
|
||||
|
||||
For more info:
|
||||
|
||||
- [AbortController constructor](https://developer.mozilla.org/en-US/docs/Web/API/AbortController)
|
||||
- [AbortSignal: timeout() static method](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static)
|
||||
|
||||
|
||||
### `extractFromJson()`
|
||||
|
||||
Extract feed data from JSON string.
|
||||
Return an object which contains feed data.
|
||||
|
||||
#### Syntax
|
||||
|
||||
```ts
|
||||
extractFromJson(String json)
|
||||
extractFromJson(String json, Object parserOptions)
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
import { extractFromJson } from '@extractus/feed-extractor'
|
||||
|
||||
const url = 'https://www.jsonfeed.org/feed.json'
|
||||
// this resource provides data in JSON feed format
|
||||
// so we fetch remote content as json
|
||||
// then pass to feed-extractor
|
||||
const res = await fetch(url)
|
||||
const json = await res.json()
|
||||
|
||||
const feed = extractFromJson(json)
|
||||
console.log(feed)
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
##### `json` *required*
|
||||
|
||||
JSON string loaded from JSON feed resource.
|
||||
|
||||
##### `parserOptions` *optional*
|
||||
|
||||
See [parserOptions](#parseroptions-optional) above.
|
||||
|
||||
|
||||
### `extractFromXml()`
|
||||
|
||||
Extract feed data from XML string.
|
||||
Return an object which contains feed data.
|
||||
|
||||
#### Syntax
|
||||
|
||||
```ts
|
||||
extractFromXml(String xml)
|
||||
extractFromXml(String xml, Object parserOptions)
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```js
|
||||
import { extractFromXml } from '@extractus/feed-extractor'
|
||||
|
||||
const url = 'https://news.google.com/atom'
|
||||
// this resource provides data in ATOM feed format
|
||||
// so we fetch remote content as text
|
||||
// then pass to feed-extractor
|
||||
const res = await fetch(url)
|
||||
const xml = await res.text()
|
||||
|
||||
const feed = extractFromXml(xml)
|
||||
console.log(feed)
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
##### `xml` *required*
|
||||
|
||||
XML string loaded from RSS/ATOM feed resource.
|
||||
|
||||
##### `parserOptions` *optional*
|
||||
|
||||
See [parserOptions](#parseroptions-optional) above.
|
||||
|
||||
|
||||
## Test
|
||||
|
||||
```bash
|
||||
git clone https://github.com/extractus/feed-extractor.git
|
||||
cd feed-extractor
|
||||
pnpm i
|
||||
pnpm test
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
## Quick evaluation
|
||||
|
||||
```bash
|
||||
git clone https://github.com/extractus/feed-extractor.git
|
||||
cd feed-extractor
|
||||
pnpm i
|
||||
pnpm eval https://news.google.com/rss
|
||||
```
|
||||
|
||||
## License
|
||||
The MIT License (MIT)
|
||||
|
||||
## Support the project
|
||||
|
||||
If you find value from this open source project, you can support in the following ways:
|
||||
|
||||
- Give it a star ⭐
|
||||
- Buy me a coffee: https://paypal.me/ndaidong 🍵
|
||||
- Subscribe [Feed Reader service](https://rapidapi.com/pwshub-pwshub-default/api/feed-reader1/) on RapidAPI 😉
|
||||
|
||||
Thank you.
|
||||
|
||||
---
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
Due to resource limitations, only the latest stable minor release is getting bugfixes (including security ones).
|
||||
|
||||
So e.g. if the latest stable version is 6.1.3, then 6.1.x line will still get security fixes but older versions (like 6.0.x) won't get any fixes.
|
||||
|
||||
Description above is a general rule and may be altered on case by case basis.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
You can report low severity vulnerabilities as GitHub issues.
|
||||
|
||||
More severe vulnerabilities should be reported to email extractus.security@skiff.com.
|
||||
|
||||
---
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"imports": {
|
||||
"cross-fetch": "./src/deno/cross-fetch.js",
|
||||
"html-entities": "https://esm.sh/html-entities@2.4.0",
|
||||
"fast-xml-parser": "https://esm.sh/fast-xml-parser@4.3.2",
|
||||
"bellajs": "https://esm.sh/bellajs@11.1.2"
|
||||
}
|
||||
}
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
// eval.js
|
||||
// to quickly test with a single url or file
|
||||
|
||||
import { extract } from './src/main.js'
|
||||
|
||||
const run = async (url) => {
|
||||
try {
|
||||
console.time('extract-feed')
|
||||
const feed = await extract(url)
|
||||
console.log(feed)
|
||||
console.timeEnd('extract-feed')
|
||||
} catch (err) {
|
||||
console.log(err.message)
|
||||
}
|
||||
}
|
||||
|
||||
const init = (argv) => {
|
||||
if (argv.length === 3) {
|
||||
const url = argv[2]
|
||||
return run(url)
|
||||
}
|
||||
return 'Nothing to do!'
|
||||
}
|
||||
|
||||
init(process.argv)
|
||||
-95
@@ -1,95 +0,0 @@
|
||||
// Type definitions
|
||||
|
||||
export interface FeedEntry {
|
||||
/**
|
||||
* id, guid, or generated identifier for the entry
|
||||
*/
|
||||
id: string;
|
||||
link?: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
published?: Date;
|
||||
}
|
||||
|
||||
export interface FeedData {
|
||||
link?: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
generator?: string;
|
||||
language?: string;
|
||||
published?: Date;
|
||||
entries?: Array<FeedEntry>;
|
||||
}
|
||||
|
||||
export interface ProxyConfig {
|
||||
target?: string;
|
||||
headers?: any;
|
||||
}
|
||||
|
||||
export interface ReaderOptions {
|
||||
/**
|
||||
* normalize feed data or keep original
|
||||
* default: true
|
||||
*/
|
||||
normalization?: boolean;
|
||||
/**
|
||||
* convert datetime to ISO format
|
||||
* default: true
|
||||
*/
|
||||
useISODateFormat?: boolean;
|
||||
/**
|
||||
* to truncate description
|
||||
* default: 210
|
||||
*/
|
||||
descriptionMaxLen?: number;
|
||||
/**
|
||||
* fast-xml-parser options
|
||||
* https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md
|
||||
*/
|
||||
xmlParserOptions?: any;
|
||||
/**
|
||||
* fill in the baseurl when it does not exist in the link
|
||||
* default: ''
|
||||
*/
|
||||
baseUrl?: string;
|
||||
/**
|
||||
* merge extra feed fields in result
|
||||
*/
|
||||
getExtraFeedFields?: (feedData: object) => object;
|
||||
/**
|
||||
* merge extra entry fields in result
|
||||
*/
|
||||
getExtraEntryFields?: (entryData: object) => object;
|
||||
}
|
||||
|
||||
export interface FetchOptions {
|
||||
// Definitions by: Ryan Graham <https://github.com/ryan-codingintrigue>
|
||||
method?: "GET" | "POST" | "DELETE" | "PATCH" | "PUT" | "HEAD" | "OPTIONS" | "CONNECT";
|
||||
headers?: any;
|
||||
body?: any;
|
||||
mode?: "cors" | "no-cors" | "same-origin";
|
||||
credentials?: "omit" | "same-origin" | "include";
|
||||
cache?: "default" | "no-store" | "reload" | "no-cache" | "force-cache" | "only-if-cached";
|
||||
redirect?: "follow" | "error" | "manual";
|
||||
referrer?: string;
|
||||
referrerPolicy?: "referrer" | "no-referrer-when-downgrade" | "origin" | "origin-when-cross-origin" | "unsafe-url";
|
||||
integrity?: any;
|
||||
proxy?: ProxyConfig;
|
||||
/**
|
||||
* http proxy agent
|
||||
* default: null
|
||||
*/
|
||||
agent?: object;
|
||||
/**
|
||||
* signal to terminate request
|
||||
* default: null
|
||||
*/
|
||||
signal?: object;
|
||||
}
|
||||
|
||||
export function extractFromXml(xml: string, options?: ReaderOptions): FeedData;
|
||||
export function extractFromJson(json: string, options?: ReaderOptions): FeedData;
|
||||
|
||||
export function extract(url: string, options?: ReaderOptions, fetchOptions?: FetchOptions): Promise<FeedData>;
|
||||
|
||||
export function read(url: string, options?: ReaderOptions, fetchOptions?: FetchOptions): Promise<FeedData>;
|
||||
-1
@@ -1 +0,0 @@
|
||||
../fast-xml-parser/src/cli/cli.js
|
||||
-572
@@ -1,572 +0,0 @@
|
||||
Note: If you find missing information about particular minor version, that version must have been changed without any functional change in this library.
|
||||
|
||||
**4.3.2 / 2023-10-02**
|
||||
* fix `jObj.hasOwnProperty` when give input is null (By [Arda TANRIKULU](https://github.com/ardatan))
|
||||
|
||||
**4.3.1 / 2023-09-24**
|
||||
* revert back "Fix typings for builder and parser to make return type generic" to avoid failure of existing projects. Need to decide a common approach.
|
||||
|
||||
**4.3.0 / 2023-09-20**
|
||||
* Fix stopNodes to work with removeNSPrefix (#607) (#608) (By [Craig Andrews]https://github.com/candrews))
|
||||
* Fix #610 ignore properties set to Object.prototype
|
||||
* Fix typings for builder and parser to make return type generic (By [Sarah Dayan](https://github.com/sarahdayan))
|
||||
|
||||
**4.2.7 / 2023-07-30**
|
||||
* Fix: builder should set text node correctly when only textnode is present (#589) (By [qianqing](https://github.com/joneqian))
|
||||
* Fix: Fix for null and undefined attributes when building xml (#585) (#598). A null or undefined value should be ignored. (By [Eugenio Ceschia](https://github.com/cecia234))
|
||||
|
||||
**4.2.6 / 2023-07-17**
|
||||
* Fix: Remove trailing slash from jPath for self-closing tags (#595) (By [Maciej Radzikowski](https://github.com/m-radzikowski))
|
||||
|
||||
**4.2.5 / 2023-06-22**
|
||||
* change code implementation
|
||||
|
||||
**4.2.4 / 2023-06-06**
|
||||
* fix security bug
|
||||
|
||||
**4.2.3 / 2023-06-05**
|
||||
* fix security bug
|
||||
|
||||
**4.2.2 / 2023-04-18**
|
||||
* fix #562: fix unpaired tag when it comes in last of a nested tag. Also throw error when unpaired tag is used as closing tag
|
||||
|
||||
**4.2.1 / 2023-04-18**
|
||||
* fix: jpath after unpaired tags
|
||||
|
||||
**4.2.0 / 2023-04-09**
|
||||
* support `updateTag` parser property
|
||||
|
||||
**4.1.4 / 2023-04-08**
|
||||
* update typings to let user create XMLBuilder instance without options (#556) (By [Patrick](https://github.com/omggga))
|
||||
* fix: IsArray option isn't parsing tags with 0 as value correctly #490 (#557) (By [Aleksandr Murashkin](https://github.com/p-kuen))
|
||||
* feature: support `oneListGroup` to group repeated children tags udder single group
|
||||
|
||||
**4.1.3 / 2023-02-26**
|
||||
* fix #546: Support complex entity value
|
||||
|
||||
**4.1.2 / 2023-02-12**
|
||||
* Security Fix
|
||||
|
||||
**4.1.1 / 2023-02-03**
|
||||
* Fix #540: ignoreAttributes breaks unpairedTags
|
||||
* Refactor XML builder code
|
||||
|
||||
**4.1.0 / 2023-02-02**
|
||||
* Fix '<' or '>' in DTD comment throwing an error. (#533) (By [Adam Baker](https://github.com/Cwazywierdo))
|
||||
* Set "eNotation" to 'true' as default
|
||||
|
||||
**4.0.15 / 2023-01-25**
|
||||
* make "eNotation" optional
|
||||
|
||||
**4.0.14 / 2023-01-22**
|
||||
* fixed: add missed typing "eNotation" to parse values
|
||||
|
||||
**4.0.13 / 2023-01-07**
|
||||
* preserveorder formatting (By [mdeknowis](https://github.com/mdeknowis))
|
||||
* support `transformAttributeName` (By [Erik Rothoff Andersson](https://github.com/erkie))
|
||||
|
||||
**4.0.12 / 2022-11-19**
|
||||
* fix typescript
|
||||
|
||||
**4.0.11 / 2022-10-05**
|
||||
* fix #501: parse for entities only once
|
||||
|
||||
**4.0.10 / 2022-09-14**
|
||||
* fix broken links in demo site (By [Yannick Lang](https://github.com/layaxx))
|
||||
* fix #491: tagValueProcessor type definition (By [Andrea Francesco Speziale](https://github.com/andreafspeziale))
|
||||
* Add jsdocs for tagValueProcessor
|
||||
|
||||
|
||||
**4.0.9 / 2022-07-10**
|
||||
* fix #470: stop-tag can have self-closing tag with same name
|
||||
* fix #472: stopNode can have any special tag inside
|
||||
* Allow !ATTLIST and !NOTATION with DOCTYPE
|
||||
* Add transformTagName option to transform tag names when parsing (#469) (By [Erik Rothoff Andersson](https://github.com/erkie))
|
||||
|
||||
**4.0.8 / 2022-05-28**
|
||||
* Fix CDATA parsing returning empty string when value = 0 (#451) (By [ndelanou](https://github.com/ndelanou))
|
||||
* Fix stopNodes when same tag appears inside node (#456) (By [patrickshipe](https://github.com/patrickshipe))
|
||||
* fix #468: prettify own properties only
|
||||
|
||||
**4.0.7 / 2022-03-18**
|
||||
* support CDATA even if tag order is not preserved
|
||||
* support Comments even if tag order is not preserved
|
||||
* fix #446: XMLbuilder should not indent XML declaration
|
||||
|
||||
**4.0.6 / 2022-03-08**
|
||||
* fix: call tagValueProcessor only once for array items
|
||||
* fix: missing changed for #437
|
||||
|
||||
**4.0.5 / 2022-03-06**
|
||||
* fix #437: call tagValueProcessor from XML builder
|
||||
|
||||
**4.0.4 / 2022-03-03**
|
||||
* fix #435: should skip unpaired and self-closing nodes when set as stopnodes
|
||||
|
||||
**4.0.3 / 2022-02-15**
|
||||
* fix: ReferenceError when Bundled with Strict (#431) (By [Andreas Heissenberger](https://github.com/aheissenberger))
|
||||
|
||||
|
||||
**4.0.2 / 2022-02-04**
|
||||
* builder supports `suppressUnpairedNode`
|
||||
* parser supports `ignoreDeclaration` and `ignorePiTags`
|
||||
* fix: when comment is parsed as text value if given as `<!--> ...` #423
|
||||
* builder supports decoding `&`
|
||||
|
||||
**4.0.1 / 2022-01-08**
|
||||
* fix builder for pi tag
|
||||
* fix: support suppressBooleanAttrs by builder
|
||||
|
||||
**4.0.0 / 2022-01-06**
|
||||
* Generating different combined, parser only, builder only, validator only browser bundles
|
||||
* Keeping cjs modules as they can be imported in cjs and esm modules both. Otherwise refer `esm` branch.
|
||||
|
||||
**4.0.0-beta.8 / 2021-12-13**
|
||||
* call tagValueProcessor for stop nodes
|
||||
|
||||
**4.0.0-beta.7 / 2021-12-09**
|
||||
* fix Validator bug when an attribute has no value but '=' only
|
||||
* XML Builder should suppress unpaired tags by default.
|
||||
* documents update for missing features
|
||||
* refactoring to use Object.assign
|
||||
* refactoring to remove repeated code
|
||||
|
||||
**4.0.0-beta.6 / 2021-12-05**
|
||||
* Support PI Tags processing
|
||||
* Support `suppressBooleanAttributes` by XML Builder for attributes with value `true`.
|
||||
|
||||
**4.0.0-beta.5 / 2021-12-04**
|
||||
* fix: when a tag with name "attributes"
|
||||
|
||||
**4.0.0-beta.4 / 2021-12-02**
|
||||
* Support HTML document parsing
|
||||
* skip stop nodes parsing when building the XML from JS object
|
||||
* Support external entites without DOCTYPE
|
||||
* update dev dependency: strnum v1.0.5 to fix long number issue
|
||||
|
||||
**4.0.0-beta.3 / 2021-11-30**
|
||||
* support global stopNodes expression like "*.stop"
|
||||
* support self-closing and paired unpaired tags
|
||||
* fix: CDATA should not be parsed.
|
||||
* Fix typings for XMLBuilder (#396)(By [Anders Emil Salvesen](https://github.com/andersem))
|
||||
* supports XML entities, HTML entities, DOCTYPE entities
|
||||
|
||||
**⚠️ 4.0.0-beta.2 / 2021-11-19**
|
||||
* rename `attrMap` to `attibutes` in parser output when `preserveOrder:true`
|
||||
* supports unpairedTags
|
||||
|
||||
**⚠️ 4.0.0-beta.1 / 2021-11-18**
|
||||
* Parser returns an array now
|
||||
* to make the structure common
|
||||
* and to return root level detail
|
||||
* renamed `cdataTagName` to `cdataPropName`
|
||||
* Added `commentPropName`
|
||||
* fix typings
|
||||
|
||||
**⚠️ 4.0.0-beta.0 / 2021-11-16**
|
||||
* Name change of many configuration properties.
|
||||
* `attrNodeName` to `attributesGroupName`
|
||||
* `attrValueProcessor` to `attributeValueProcessor`
|
||||
* `parseNodeValue` to `parseTagValue`
|
||||
* `ignoreNameSpace` to `removeNSPrefix`
|
||||
* `numParseOptions` to `numberParseOptions`
|
||||
* spelling correction for `suppressEmptyNode`
|
||||
* Name change of cli and browser bundle to **fxparser**
|
||||
* `isArray` option is added to parse a tag into array
|
||||
* `preserveOrder` option is added to render XML in such a way that the result js Object maintains the order of properties same as in XML.
|
||||
* Processing behaviour of `tagValueProcessor` and `attributeValueProcessor` are changes with extra input parameters
|
||||
* j2xparser is renamed to XMLBuilder.
|
||||
* You need to build XML parser instance for given options first before parsing XML.
|
||||
* fix #327, #336: throw error when extra text after XML content
|
||||
* fix #330: attribute value can have '\n',
|
||||
* fix #350: attrbiutes can be separated by '\n' from tagname
|
||||
|
||||
3.21.1 / 2021-10-31
|
||||
* Correctly format JSON elements with a text prop but no attribute props ( By [haddadnj](https://github.com/haddadnj) )
|
||||
|
||||
3.21.0 / 2021-10-25
|
||||
* feat: added option `rootNodeName` to set tag name for array input when converting js object to XML.
|
||||
* feat: added option `alwaysCreateTextNode` to force text node creation (by: *@massimo-ua*)
|
||||
* ⚠️ feat: Better error location for unclosed tags. (by *@Gei0r*)
|
||||
* Some error messages would be changed when validating XML. Eg
|
||||
* `{ InvalidXml: "Invalid '[ \"rootNode\"]' found." }` → `{InvalidTag: "Unclosed tag 'rootNode'."}`
|
||||
* `{ InvalidTag: "Closing tag 'rootNode' is expected inplace of 'rootnode'." }` → `{ InvalidTag: "Expected closing tag 'rootNode' (opened in line 1) instead of closing tag 'rootnode'."}`
|
||||
* ⚠️ feat: Column in error response when validating XML
|
||||
```js
|
||||
{
|
||||
"code": "InvalidAttr",
|
||||
"msg": "Attribute 'abc' is repeated.",
|
||||
"line": 1,
|
||||
"col": 22
|
||||
}
|
||||
```
|
||||
|
||||
3.20.1 / 2021-09-25
|
||||
* update strnum package
|
||||
|
||||
3.20.0 / 2021-09-10
|
||||
* Use strnum npm package to parse string to number
|
||||
* breaking change: long number will be parsed to scientific notation.
|
||||
|
||||
3.19.0 / 2021-03-14
|
||||
* License changed to MIT original
|
||||
* Fix #321 : namespace tag parsing
|
||||
|
||||
3.18.0 / 2021-02-05
|
||||
* Support RegEx and function in arrayMode option
|
||||
* Fix #317 : validate nested PI tags
|
||||
|
||||
3.17.4 / 2020-06-07
|
||||
* Refactor some code to support IE11
|
||||
* Fix: `<tag >` space as attribute string
|
||||
|
||||
3.17.3 / 2020-05-23
|
||||
* Fix: tag name separated by \n \t
|
||||
* Fix: throw error for unclosed tags
|
||||
|
||||
3.17.2 / 2020-05-23
|
||||
* Fixed an issue in processing doctype tag
|
||||
* Fixed tagName where it should not have whitespace chars
|
||||
|
||||
3.17.1 / 2020-05-19
|
||||
* Fixed an issue in checking opening tag
|
||||
|
||||
3.17.0 / 2020-05-18
|
||||
* parser: fix '<' issue when it comes in aatr value
|
||||
* parser: refactoring to remove dependency from regex
|
||||
* validator: fix IE 11 issue for error messages
|
||||
* updated dev dependencies
|
||||
* separated benchmark module to sub-module
|
||||
* breaking change: comments will not be removed from CDATA data
|
||||
|
||||
3.16.0 / 2020-01-12
|
||||
* validaor: fix for ampersand characters (#215)
|
||||
* refactoring to support unicode chars in tag name
|
||||
* update typing for validator error
|
||||
|
||||
3.15.1 / 2019-12-09
|
||||
* validaor: fix multiple roots are not allowed
|
||||
|
||||
3.15.0 / 2019-11-23
|
||||
* validaor: improve error messaging
|
||||
* validator: add line number in case of error
|
||||
* validator: add more error scenarios to make it more descriptive
|
||||
|
||||
3.14.0 / 2019-10-25
|
||||
* arrayMode for XML to JS obj parsing
|
||||
|
||||
3.13.0 / 2019-10-02
|
||||
* pass tag/attr name to tag/attr value processor
|
||||
* inbuilt optional validation with XML parser
|
||||
|
||||
3.12.21 / 2019-10-02
|
||||
* Fix validator for unclosed XMLs
|
||||
* move nimnjs dependency to dev dependency
|
||||
* update dependencies
|
||||
|
||||
3.12.20 / 2019-08-16
|
||||
* Revert: Fix #167: '>' in attribute value as it is causing high performance degrade.
|
||||
|
||||
3.12.19 / 2019-07-28
|
||||
* Fix js to xml parser should work for date values. (broken: `tagValueProcessor` will receive the original value instead of string always) (breaking change)
|
||||
|
||||
3.12.18 / 2019-07-27
|
||||
* remove configstore dependency
|
||||
|
||||
3.12.17 / 2019-07-14
|
||||
* Fix #167: '>' in attribute value
|
||||
|
||||
3.12.16 / 2019-03-23
|
||||
* Support a new option "stopNodes". (#150)
|
||||
Accept the list of tags which are not required to be parsed. Instead, all the nested tag and data will be assigned as string.
|
||||
* Don't show post-install message
|
||||
|
||||
3.12.12 / 2019-01-11
|
||||
* fix : IE parseInt, parseFloat error
|
||||
|
||||
3.12.11 / 2018-12-24
|
||||
* fix #132: "/" should not be parsed as boolean attr in case of self closing tags
|
||||
|
||||
3.12.9 / 2018-11-23
|
||||
* fix #129 : validator should not fail when an atrribute name is 'length'
|
||||
|
||||
3.12.8 / 2018-11-22
|
||||
* fix #128 : use 'attrValueProcessor' to process attribute value in json2xml parser
|
||||
|
||||
3.12.6 / 2018-11-10
|
||||
* Fix #126: check for type
|
||||
|
||||
3.12.4 / 2018-09-12
|
||||
* Fix: include tasks in npm package
|
||||
|
||||
3.12.3 / 2018-09-12
|
||||
* Fix CLI issue raised in last PR
|
||||
|
||||
3.12.2 / 2018-09-11
|
||||
* Fix formatting for JSON to XML output
|
||||
* Migrate to webpack (PR merged)
|
||||
* fix cli (PR merged)
|
||||
|
||||
3.12.0 / 2018-08-06
|
||||
* Support hexadecimal values
|
||||
* Support true number parsing
|
||||
|
||||
3.11.2 / 2018-07-23
|
||||
* Update Demo for more options
|
||||
* Update license information
|
||||
* Update readme for formatting, users, and spelling mistakes
|
||||
* Add missing typescript definition for j2xParser
|
||||
* refactoring: change filenames
|
||||
|
||||
3.11.1 / 2018-06-05
|
||||
* fix #93: read the text after self closing tag
|
||||
|
||||
3.11.0 / 2018-05-20
|
||||
* return defaultOptions if there are not options in buildOptions function
|
||||
* added localeRange declaration in parser.d.ts
|
||||
* Added support of cyrillic characters in validator XML
|
||||
* fixed bug in validator work when XML data with byte order marker
|
||||
|
||||
3.10.0 / 2018-05-13
|
||||
* Added support of cyrillic characters in parsing XML to JSON
|
||||
|
||||
3.9.11 / 2018-05-09
|
||||
* fix https://github.com/NaturalIntelligence/fast-xml-parser/issues/80 fix nimn chars
|
||||
* update package information
|
||||
* fix https://github.com/NaturalIntelligence/fast-xml-parser/issues/86: json 2 xml parser : property with null value should be parsed to self closing tag.
|
||||
* update online demo
|
||||
* revert zombiejs to old version to support old version of node
|
||||
* update dependencies
|
||||
|
||||
3.3.10 / 2018-04-23
|
||||
* fix #77 : parse even if closing tag has space before '>'
|
||||
* include all css & js lib in demo app
|
||||
* remove babel dependencies until needed
|
||||
|
||||
3.3.9 / 2018-04-18
|
||||
* fix #74 : TS2314 TypeScript compiler error
|
||||
|
||||
3.3.8 / 2018-04-17
|
||||
* fix #73 : IE doesn't support Object.assign
|
||||
|
||||
3.3.7 / 2018-04-14
|
||||
* fix: use let insted of const in for loop of validator
|
||||
* Merge pull request
|
||||
https://github.com/NaturalIntelligence/fast-xml-parser/issues/71 from bb/master
|
||||
first draft of typings for typescript
|
||||
https://github.com/NaturalIntelligence/fast-xml-parser/issues/69
|
||||
* Merge pull request
|
||||
https://github.com/NaturalIntelligence/fast-xml-parser/issues/70 from bb/patch-1
|
||||
fix some typos in readme
|
||||
|
||||
3.3.6 / 2018-03-21
|
||||
* change arrow functions to full notation for IE compatibility
|
||||
|
||||
3.3.5 / 2018-03-15
|
||||
* fix https://github.com/NaturalIntelligence/fast-xml-parser/issues/67 : attrNodeName invalid behavior
|
||||
* fix: remove decodeHTML char condition
|
||||
|
||||
3.3.4 / 2018-03-14
|
||||
* remove dependency on "he" package
|
||||
* refactor code to separate methods in separate files.
|
||||
* draft code for transforming XML to json string. It is not officially documented due to performance issue.
|
||||
|
||||
3.3.0 / 2018-03-05
|
||||
* use common default options for XML parsing for consistency. And add `parseToNimn` method.
|
||||
* update nexttodo
|
||||
* update README about XML to Nimn transformation and remove special notes about 3.x release
|
||||
* update CONTRIBUTING.ms mentioning nexttodo
|
||||
* add negative case for XML PIs
|
||||
* validate xml processing instruction tags https://github.com/NaturalIntelligence/fast-xml-parser/issues/62
|
||||
* nimndata: handle array with object
|
||||
* nimndata: node with nested node and text node
|
||||
* nimndata: handle attributes and text node
|
||||
* nimndata: add options, handle array
|
||||
* add xml to nimn data converter
|
||||
* x2j: direct access property with tagname
|
||||
* update changelog
|
||||
* fix validator when single quote presents in value enclosed with double quotes or vice versa
|
||||
* Revert "remove unneded nimnjs dependency, move opencollective to devDependencies and replace it
|
||||
with more light opencollective-postinstall"
|
||||
This reverts commit d47aa7181075d82db4fee97fd8ea32b056fe3f46.
|
||||
* Merge pull request: https://github.com/NaturalIntelligence/fast-xml-parser/issues/63 from HaroldPutman/suppress-undefined
|
||||
Keep undefined nodes out of the XML output : This is useful when you are deleting nodes from the JSON and rewriting XML.
|
||||
|
||||
3.2.4 / 2018-03-01
|
||||
* fix #59 fix in validator when open quote presents in attribute value
|
||||
* Create nexttodo.md
|
||||
* exclude static from bitHound tests
|
||||
* add package lock
|
||||
|
||||
3.2.3 / 2018-02-28
|
||||
* Merge pull request from Delagen/master: fix namespaces can contain the same characters as xml names
|
||||
|
||||
3.2.2 / 2018-02-22
|
||||
* fix: attribute xmlns should not be removed if ignoreNameSpace is false
|
||||
* create CONTRIBUTING.md
|
||||
|
||||
3.2.1 / 2018-02-17
|
||||
* fix: empty attribute should be parsed
|
||||
|
||||
3.2.0 / 2018-02-16
|
||||
* Merge pull request : Dev to Master
|
||||
* Update README and version
|
||||
* j2x:add performance test
|
||||
* j2x: Remove extra empty line before closing tag
|
||||
* j2x: suppress empty nodes to self closing node if configured
|
||||
* j2x: provide option to give indentation depth
|
||||
* j2x: make optional formatting
|
||||
* j2x: encodeHTMLchat
|
||||
* j2x: handle cdata tag
|
||||
* j2x: handle grouped attributes
|
||||
* convert json to xml
|
||||
- nested object
|
||||
- array
|
||||
- attributes
|
||||
- text value
|
||||
* small refactoring
|
||||
* Merge pull request: Update cli.js to let user validate XML file or data
|
||||
* Add option for rendering CDATA as separate property
|
||||
|
||||
3.0.1 / 2018-02-09
|
||||
* fix CRLF: replace it with single space in attributes value only.
|
||||
|
||||
3.0.0 / 2018-02-08
|
||||
* change online tool with new changes
|
||||
* update info about new options
|
||||
* separate tag value processing to separate function
|
||||
* make HTML decoding optional
|
||||
* give an option to allow boolean attributes
|
||||
* change cli options as per v3
|
||||
* Correct comparison table format on README
|
||||
* update v3 information
|
||||
* some performance improvement changes
|
||||
* Make regex object local to the method and move some common methods to util
|
||||
* Change parser to
|
||||
- handle multiple instances of CDATA
|
||||
- make triming of value optionals
|
||||
- HTML decode attribute and text value
|
||||
- refactor code to separate files
|
||||
* Ignore newline chars without RE (in validator)
|
||||
* validate for XML prolog
|
||||
* Validate DOCTYPE without RE
|
||||
* Update validator to return error response
|
||||
* Update README to add detail about V3
|
||||
* Separate xmlNode model class
|
||||
* include vscode debug config
|
||||
* fix for repeated object
|
||||
* fix attribute regex for boolean attributes
|
||||
* Fix validator for invalid attributes
|
||||
2.9.4 / 2018-02-02
|
||||
* Merge pull request: Decode HTML characters
|
||||
* refactor source folder name
|
||||
* ignore bundle / browser js to be published to npm
|
||||
2.9.3 / 2018-01-26
|
||||
* Merge pull request: Correctly remove CRLF line breaks
|
||||
* Enable to parse attribute in online editor
|
||||
* Fix testing demo app test
|
||||
* Describe parsing options
|
||||
* Add options for online demo
|
||||
2.9.2 / 2018-01-18
|
||||
* Remove check if tag starting with "XML"
|
||||
* Fix: when there are spaces before / after CDATA
|
||||
|
||||
2.9.1 / 2018-01-16
|
||||
* Fix: newline should be replaced with single space
|
||||
* Fix: for single and multiline comments
|
||||
* validate xml with CDATA
|
||||
* Fix: the issue when there is no space between 2 attributes
|
||||
* Fix: https://github.com/NaturalIntelligence/fast-xml-parser/issues/33: when there is newline char in attr val, it doesn't parse
|
||||
* Merge pull request: fix ignoreNamespace
|
||||
* fix: don't wrap attributes if only namespace attrs
|
||||
* fix: use portfinder for run tests, update deps
|
||||
* fix: don't treat namespaces as attributes when ignoreNamespace enabled
|
||||
|
||||
2.9.0 / 2018-01-10
|
||||
* Rewrite the validator to handle large files.
|
||||
Ignore DOCTYPE validation.
|
||||
* Fix: When attribute value has equal sign
|
||||
|
||||
2.8.3 / 2017-12-15
|
||||
* Fix: when a tag has value along with subtags
|
||||
|
||||
2.8.2 / 2017-12-04
|
||||
* Fix value parsing for IE
|
||||
|
||||
2.8.1 / 2017-12-01
|
||||
* fix: validator should return false instead of err when invalid XML
|
||||
|
||||
2.8.0 / 2017-11-29
|
||||
* Add CLI option to ignore value conversion
|
||||
* Fix variable name when filename is given on CLI
|
||||
* Update CLI help text
|
||||
* Merge pull request: xml2js: Accept standard input
|
||||
* Test Node 8
|
||||
* Update dependencies
|
||||
* Bundle readToEnd
|
||||
* Add ability to read from standard input
|
||||
|
||||
2.7.4 / 2017-09-22
|
||||
* Merge pull request: Allow wrap attributes with subobject to compatible with other parsers output
|
||||
|
||||
2.7.3 / 2017-08-02
|
||||
* fix: handle CDATA with regx
|
||||
|
||||
2.7.2 / 2017-07-30
|
||||
* Change travis config for yarn caching
|
||||
* fix validator: when tag property is same as array property
|
||||
* Merge pull request: Failing test case in validator for valid SVG
|
||||
|
||||
2.7.1 / 2017-07-26
|
||||
* Fix: Handle val 0
|
||||
|
||||
2.7.0 / 2017-07-25
|
||||
* Fix test for arrayMode
|
||||
* Merge pull request: Add arrayMode option to parse any nodes as arrays
|
||||
|
||||
2.6.0 / 2017-07-14
|
||||
* code improvement
|
||||
* Add unit tests for value conversion for attr
|
||||
* Merge pull request: option of an attribute value conversion to a number (textAttrConversion) the same way as the textNodeConversion option does. Default value is false.
|
||||
|
||||
2.5.1 / 2017-07-01
|
||||
* Fix XML element name pattern
|
||||
* Fix XML element name pattern while parsing
|
||||
* Fix validation for xml tag element
|
||||
|
||||
2.5.0 / 2017-06-25
|
||||
* Improve Validator performance
|
||||
* update attr matching regex
|
||||
* Add perf tests
|
||||
* Improve atrr regex to handle all cases
|
||||
|
||||
2.4.4 / 2017-06-08
|
||||
* Bug fix: when an attribute has single or double quote in value
|
||||
|
||||
2.4.3 / 2017-06-05
|
||||
* Bug fix: when multiple CDATA tags are given
|
||||
* Merge pull request: add option "textNodeConversion"
|
||||
* add option "textNodeConversion"
|
||||
|
||||
2.4.1 / 2017-04-14
|
||||
* fix tests
|
||||
* Bug fix: preserve initial space of node value
|
||||
* Handle CDATA
|
||||
|
||||
2.3.1 / 2017-03-15
|
||||
* Bug fix: when single self closing tag
|
||||
* Merge pull request: fix .codeclimate.yml
|
||||
* Update .codeclimate.yml - Fixed config so it does not error anymore.
|
||||
* Update .codeclimate.yml
|
||||
|
||||
2.3.0 / 2017-02-26
|
||||
* Code improvement
|
||||
* add bithound config
|
||||
* Update usage
|
||||
* Update travis to generate bundle js before running tests
|
||||
* 1.Browserify, 2. add more tests for validator
|
||||
* Add validator
|
||||
* Fix CLI default parameter bug
|
||||
|
||||
2.2.1 / 2017-02-05
|
||||
* Bug fix: CLI default option
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2017 Amit Kumar Gupta
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
-214
@@ -1,214 +0,0 @@
|
||||
# [fast-xml-parser](https://www.npmjs.com/package/fast-xml-parser)
|
||||
[](https://snyk.io/test/github/naturalintelligence/fast-xml-parser)
|
||||
[![NPM quality][quality-image]][quality-url]
|
||||
[](https://coveralls.io/github/NaturalIntelligence/fast-xml-parser?branch=master)
|
||||
[<img src="https://img.shields.io/badge/Try-me-blue.svg?colorA=FFA500&colorB=0000FF" alt="Try me"/>](https://naturalintelligence.github.io/fast-xml-parser/)
|
||||
[](https://npm.im/fast-xml-parser)
|
||||
|
||||
[quality-image]: http://npm.packagequality.com/shield/fast-xml-parser.svg?style=flat-square
|
||||
[quality-url]: http://packagequality.com/#?package=fast-xml-parser
|
||||
|
||||
|
||||
Validate XML, Parse XML to JS Object, or Build XML from JS Object without C/C++ based libraries and no callback.
|
||||
|
||||
<font size="6">I need a Career advice. I've posted the query on my <a href="github.com/amitguptagwl">profile</a>. Your support would be appreciable.</font>
|
||||
|
||||
Sponsor this project 👉
|
||||
<a href="https://github.com/sponsors/NaturalIntelligence">
|
||||
<img src="https://raw.githubusercontent.com/NaturalIntelligence/ThankYouBackers/main/github_sponsor.png" width="200" />
|
||||
</a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/donate" target="_blank">
|
||||
<img src="https://opencollective.com/fast-xml-parser/donate/button@2x.png?color=blue" width=200 />
|
||||
</a>
|
||||
<a href="https://paypal.me/naturalintelligence"> <img src="static/img/support_paypal.svg" alt="Stubmatic donate button" width="200"/></a>
|
||||
|
||||
|
||||
### Current Sponsors
|
||||
|
||||
Check the complete list at [ThankYouBackers](https://github.com/NaturalIntelligence/ThankYouBackers) for our sponsors and supporters.
|
||||
|
||||
Through Github
|
||||
|
||||
<a href="https://github.com/getsentry" target="_blank"><img src="https://user-images.githubusercontent.com/7692328/204701653-ce369b29-6fdb-48f2-9848-593d04628a2b.png" width="60px"></a>
|
||||
|
||||
Through OpenCollective
|
||||
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/0/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/0/avatar.svg"></a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/1/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/1/avatar.svg"></a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/2/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/2/avatar.svg"></a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/3/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/3/avatar.svg"></a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/4/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/4/avatar.svg"></a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/5/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/5/avatar.svg"></a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/6/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/6/avatar.svg"></a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/7/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/7/avatar.svg"></a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/8/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/8/avatar.svg"></a>
|
||||
<a href="https://opencollective.com/fast-xml-parser/sponsor/9/website" target="_blank"><img src="https://opencollective.com/fast-xml-parser/sponsor/9/avatar.svg"></a>
|
||||
|
||||
|
||||
## Users
|
||||
|
||||
<a href="https://github.com/renovatebot/renovate" title="renovate" ><img src="https://avatars1.githubusercontent.com/u/38656520" width="60px" ></a>
|
||||
<a href="https://vmware.com/" title="vmware" > <img src="https://avatars0.githubusercontent.com/u/473334" width="60px" ></a>
|
||||
<a href="https://opensource.microsoft.com/" title="microsoft" > <img src="https://avatars0.githubusercontent.com/u/6154722" width="60px" ></a>
|
||||
<a href="http://ibm.github.io/" title="IBM" > <img src="https://avatars2.githubusercontent.com/u/1459110" width="60px" ></a>
|
||||
<a href="http://www.smartbear.com" title="SmartBear Software" > <img src="https://avatars2.githubusercontent.com/u/1644671" width="60px" ></a>
|
||||
<a href="http://nasa.github.io/" title="NASA" > <img src="https://avatars0.githubusercontent.com/u/848102" width="60px" ></a>
|
||||
<a href="https://github.com/prettier" title="Prettier" > <img src="https://avatars0.githubusercontent.com/u/25822731" width="60px" ></a>
|
||||
<a href="http://brain.js.org/" title="brain.js" > <img src="https://avatars2.githubusercontent.com/u/23732838" width="60px" ></a>
|
||||
<a href="https://github.com/aws" title="AWS SDK" > <img src="https://avatars.githubusercontent.com/u/2232217" width="60px" ></a>
|
||||
<a href="http://www.fda.gov/" title="Food and Drug Administration " > <img src="https://avatars2.githubusercontent.com/u/6471964" width="60px" ></a>
|
||||
<a href="http://www.magento.com/" title="Magento" > <img src="https://avatars2.githubusercontent.com/u/168457" width="60px" ></a>
|
||||
<a href="https://github.com/SAP" title="SAP" > <img src="https://user-images.githubusercontent.com/7692328/204835214-d9d25b58-e3df-408d-87a3-c7d36b578ee4.png" width="60px" ></a>
|
||||
<a href="https://github.com/postmanlabs" title="postman" > <img src="https://user-images.githubusercontent.com/7692328/204835529-e9e290ad-696a-49ad-9d34-08e955704715.png" width="60px" ></a>
|
||||
<a href="https://github.com/react-native-community" title="React Native Community" > <img src="https://avatars.githubusercontent.com/u/20269980?v=4" width="60px" ></a>
|
||||
|
||||
Check the bigger [list](./USERs.md)
|
||||
|
||||
<small>The list of users is collected either from the list published by Github, communicated directly through mails/chat , or from other resources. If you feel that your name in the above list is incorrectly published or you're not the user of this library anymore then you can inform us to remove it. We'll do the necessary changes ASAP.</small>
|
||||
|
||||
If you want to be an anonymous user of this application and don't want to be highlighted anywhere then you can contact me at
|
||||
- githubissues@proton.me
|
||||
- https://linkedin.com/in/amitguptagwl/
|
||||
|
||||
## Main Features
|
||||
|
||||
<img align="right" src="static/img/fxp_logo.png" width="180px" alt="FXP logo"/>
|
||||
|
||||
* Validate XML data syntactically
|
||||
* Parse XML to JS Object
|
||||
* Build XML from JS Object
|
||||
* Works with node packages, in browser, and in CLI (press try me button above for demo)
|
||||
* Faster than any other pure JS implementation.
|
||||
* It can handle big files (tested up to 100mb).
|
||||
* Controlled parsing using various options
|
||||
* XML Entities, HTML entities, and DOCTYPE entites are supported.
|
||||
* unpaired tags (Eg `<br>` in HTML), stop nodes (Eg `<script>` in HTML) are supported.
|
||||
* You can restore almost same XML from JSON
|
||||
* Supports comments
|
||||
* It can preserve Order of tags in JS object
|
||||
* You can control if a single tag should be parsed into array.
|
||||
* Supports parsing of PI (Processing Instruction) tags with XML declaration tags
|
||||
* And many more other features.
|
||||
|
||||
## How to use
|
||||
|
||||
To use as package dependency
|
||||
`$ npm install fast-xml-parser`
|
||||
or
|
||||
`$ yarn add fast-xml-parser`
|
||||
|
||||
To use as system command
|
||||
`$ npm install fast-xml-parser -g`
|
||||
|
||||
To use it on a **webpage** include it from a [CDN](https://cdnjs.com/libraries/fast-xml-parser)
|
||||
|
||||
**Example**
|
||||
|
||||
As CLI command
|
||||
```bash
|
||||
$ fxparser some.xml
|
||||
```
|
||||
|
||||
In a node js project
|
||||
```js
|
||||
const { XMLParser, XMLBuilder, XMLValidator} = require("fast-xml-parser");
|
||||
|
||||
const parser = new XMLParser();
|
||||
let jObj = parser.parse(XMLdata);
|
||||
|
||||
const builder = new XMLBuilder();
|
||||
const xmlContent = builder.build(jObj);
|
||||
```
|
||||
|
||||
In a HTML page
|
||||
```html
|
||||
<script src="path/to/fxp.min.js"></script>
|
||||
:
|
||||
<script>
|
||||
const parser = new fxparser.XMLParser();
|
||||
parser.parse(xmlContent);
|
||||
</script>
|
||||
```
|
||||
|
||||
Check lib folder for different browser bundles
|
||||
|
||||
| Bundle Name | Size |
|
||||
| ------------------ | ---- |
|
||||
| fxbuilder.min.js | 5.2K |
|
||||
| fxparser.js | 50K |
|
||||
| fxparser.min.js | 17K |
|
||||
| fxp.min.js | 22K |
|
||||
| fxvalidator.min.js | 5.7K |
|
||||
|
||||
### Documents
|
||||
**v3**
|
||||
* [documents](./docs/v3/docs.md)
|
||||
|
||||
**v4**
|
||||
1. [GettingStarted.md](./docs/v4/1.GettingStarted.md)
|
||||
2. [XML Parser](./docs/v4/2.XMLparseOptions.md)
|
||||
3. [XML Builder](./docs/v4/3.XMLBuilder.md)
|
||||
4. [XML Validator](./docs/v4/4.XMLValidator.md)
|
||||
5. [Entities](./docs/v4/5.Entities.md)
|
||||
6. [HTML Document Parsing](./docs/v4/6.HTMLParsing.md)
|
||||
7. [PI Tag processing](./docs/v4/7.PITags.md)
|
||||
## Performance
|
||||
|
||||
### XML Parser
|
||||
|
||||
|
||||
* Y-axis: requests per second
|
||||
* X-axis: File size
|
||||
|
||||
**Large files**
|
||||
|
||||
* Y-axis: requests per second
|
||||
* X-axis: File size
|
||||
### XML Builder
|
||||
|
||||
|
||||
* Y-axis: requests per second
|
||||
|
||||
<small>negative means error</small>
|
||||
|
||||
[](https://github.com/NaturalIntelligence/ads/)
|
||||
|
||||
## Our other projects and research you must try
|
||||
|
||||
* **[BigBit standard](https://github.com/amitguptagwl/bigbit)** :
|
||||
* Single text encoding to replace UTF-8, UTF-16, UTF-32 and more with less memory.
|
||||
* Single Numeric datatype alternative of integer, float, double, long, decimal and more without precision loss.
|
||||
* **[Cytorus](https://github.com/NaturalIntelligence/cytorus)**: Be specific and flexible while running E2E tests.
|
||||
* Run tests only for a particular User Story
|
||||
* Run tests for a route or from a route
|
||||
* Customizable reporting
|
||||
* Central dashboard for better monitoring
|
||||
* Options to integrate E2E tests with Jira, Github etc using Central dashboard `Tian`.
|
||||
* **[Stubmatic](https://github.com/NaturalIntelligence/Stubmatic)** : Create fake webservices, DynamoDB or S3 servers, Manage fake/mock stub data, Or fake any HTTP(s) call.
|
||||
|
||||
|
||||
## Supporters
|
||||
### Contributors
|
||||
|
||||
This project exists thanks to [all](graphs/contributors) the people who contribute. [[Contribute](docs/CONTRIBUTING.md)].
|
||||
<!-- <a href="graphs/contributors"><img src="https://opencollective.com/fast-xml-parser/contributors.svg?width=890&button=false" /></a> -->
|
||||
<!--
|
||||
### Lead Maintainers
|
||||
|
||||
[](https://github.com/Delagen)
|
||||
|
||||
### All Contributors -->
|
||||
<a href="graphs/contributors"><img src="https://opencollective.com/fast-xml-parser/contributors.svg?width=890&button=false" /></a>
|
||||
|
||||
### Backers
|
||||
|
||||
Thank you to all our backers! 🙏 [[Become a backer](https://opencollective.com/fast-xml-parser#backer)]
|
||||
|
||||
<a href="https://opencollective.com/fast-xml-parser#backers" target="_blank"><img src="https://opencollective.com/fast-xml-parser/backers.svg?width=890"></a>
|
||||
|
||||
|
||||
|
||||
# License
|
||||
* MIT License
|
||||
|
||||
|
||||
-71
@@ -1,71 +0,0 @@
|
||||
{
|
||||
"name": "fast-xml-parser",
|
||||
"version": "4.3.2",
|
||||
"description": "Validate XML, Parse XML, Build XML without C/C++ based libraries",
|
||||
"main": "./src/fxp.js",
|
||||
"scripts": {
|
||||
"test": "nyc --reporter=lcov --reporter=text jasmine spec/*spec.js",
|
||||
"unit": "jasmine",
|
||||
"coverage": "nyc report --reporter html --reporter text -t .nyc_output --report-dir .nyc_output/summary",
|
||||
"perf": "node ./benchmark/perfTest3.js",
|
||||
"lint": "eslint src/*.js spec/*.js",
|
||||
"bundle": "webpack --config webpack-prod.config.js",
|
||||
"prettier": "prettier --write src/**/*.js",
|
||||
"publish-please": "publish-please",
|
||||
"checkReadiness": "publish-please --dry-run"
|
||||
},
|
||||
"bin": {
|
||||
"fxparser": "./src/cli/cli.js"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/NaturalIntelligence/fast-xml-parser"
|
||||
},
|
||||
"keywords": [
|
||||
"fast",
|
||||
"xml",
|
||||
"json",
|
||||
"parser",
|
||||
"xml2js",
|
||||
"x2js",
|
||||
"xml2json",
|
||||
"js",
|
||||
"cli",
|
||||
"validator",
|
||||
"validate",
|
||||
"transformer",
|
||||
"assert",
|
||||
"js2xml",
|
||||
"json2xml",
|
||||
"html"
|
||||
],
|
||||
"author": "Amit Gupta (https://solothought.com)",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@babel/core": "^7.13.10",
|
||||
"@babel/plugin-transform-runtime": "^7.13.10",
|
||||
"@babel/preset-env": "^7.13.10",
|
||||
"@babel/register": "^7.13.8",
|
||||
"babel-loader": "^8.2.2",
|
||||
"cytorus": "^0.2.9",
|
||||
"eslint": "^8.3.0",
|
||||
"he": "^1.2.0",
|
||||
"jasmine": "^3.6.4",
|
||||
"nyc": "^15.1.0",
|
||||
"prettier": "^1.19.1",
|
||||
"publish-please": "^5.5.2",
|
||||
"webpack": "^5.64.4",
|
||||
"webpack-cli": "^4.9.1"
|
||||
},
|
||||
"typings": "src/fxp.d.ts",
|
||||
"funding": [{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/NaturalIntelligence"
|
||||
},{
|
||||
"type": "paypal",
|
||||
"url": "https://paypal.me/naturalintelligence"
|
||||
}],
|
||||
"dependencies": {
|
||||
"strnum": "^1.0.5"
|
||||
}
|
||||
}
|
||||
Generated
Vendored
-93
@@ -1,93 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
/*eslint-disable no-console*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const {XMLParser, XMLValidator} = require("../fxp");
|
||||
const readToEnd = require('./read').readToEnd;
|
||||
|
||||
const version = require('./../../package.json').version;
|
||||
if (process.argv[2] === '--help' || process.argv[2] === '-h') {
|
||||
console.log(require("./man"));
|
||||
} else if (process.argv[2] === '--version') {
|
||||
console.log(version);
|
||||
} else {
|
||||
const options = {
|
||||
removeNSPrefix: true,
|
||||
ignoreAttributes: false,
|
||||
parseTagValue: true,
|
||||
parseAttributeValue: true,
|
||||
};
|
||||
let fileName = '';
|
||||
let outputFileName;
|
||||
let validate = false;
|
||||
let validateOnly = false;
|
||||
for (let i = 2; i < process.argv.length; i++) {
|
||||
if (process.argv[i] === '-ns') {
|
||||
options.removeNSPrefix = false;
|
||||
} else if (process.argv[i] === '-a') {
|
||||
options.ignoreAttributes = true;
|
||||
} else if (process.argv[i] === '-c') {
|
||||
options.parseTagValue = false;
|
||||
options.parseAttributeValue = false;
|
||||
} else if (process.argv[i] === '-o') {
|
||||
outputFileName = process.argv[++i];
|
||||
} else if (process.argv[i] === '-v') {
|
||||
validate = true;
|
||||
} else if (process.argv[i] === '-V') {
|
||||
validateOnly = true;
|
||||
} else {
|
||||
//filename
|
||||
fileName = process.argv[i];
|
||||
}
|
||||
}
|
||||
|
||||
const callback = function(xmlData) {
|
||||
let output = '';
|
||||
if (validate) {
|
||||
const parser = new XMLParser(options);
|
||||
output = parser.parse(xmlData,validate);
|
||||
} else if (validateOnly) {
|
||||
output = XMLValidator.validate(xmlData);
|
||||
process.exitCode = output === true ? 0 : 1;
|
||||
} else {
|
||||
const parser = new XMLParser(options);
|
||||
output = JSON.stringify(parser.parse(xmlData,validate), null, 4);
|
||||
}
|
||||
if (outputFileName) {
|
||||
writeToFile(outputFileName, output);
|
||||
} else {
|
||||
console.log(output);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
|
||||
if (!fileName) {
|
||||
readToEnd(process.stdin, function(err, data) {
|
||||
if (err) {
|
||||
throw err;
|
||||
}
|
||||
callback(data.toString());
|
||||
});
|
||||
} else {
|
||||
fs.readFile(fileName, function(err, data) {
|
||||
if (err) {
|
||||
throw err;
|
||||
}
|
||||
callback(data.toString());
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Seems an invalid file or stream.' + e);
|
||||
}
|
||||
}
|
||||
|
||||
function writeToFile(fileName, data) {
|
||||
fs.writeFile(fileName, data, function(err) {
|
||||
if (err) {
|
||||
throw err;
|
||||
}
|
||||
console.log('JSON output has been written to ' + fileName);
|
||||
});
|
||||
}
|
||||
Generated
Vendored
-12
@@ -1,12 +0,0 @@
|
||||
module.exports = `Fast XML Parser 4.0.0
|
||||
----------------
|
||||
$ fxparser [-ns|-a|-c|-v|-V] <filename> [-o outputfile.json]
|
||||
$ cat xmlfile.xml | fxparser [-ns|-a|-c|-v|-V] [-o outputfile.json]
|
||||
|
||||
Options
|
||||
----------------
|
||||
-ns: remove namespace from tag and atrribute name.
|
||||
-a: don't parse attributes.
|
||||
-c: parse values to premitive type.
|
||||
-v: validate before parsing.
|
||||
-V: validate only.`
|
||||
Generated
Vendored
-92
@@ -1,92 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
// Copyright 2013 Timothy J Fontaine <tjfontaine@gmail.com>
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the 'Software'), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
// THE SOFTWARE
|
||||
|
||||
/*
|
||||
|
||||
Read any stream all the way to the end and trigger a single cb
|
||||
|
||||
const http = require('http');
|
||||
|
||||
const rte = require('readtoend');
|
||||
|
||||
http.get('http://nodejs.org', function(response) {
|
||||
rte.readToEnd(response, function(err, body) {
|
||||
console.log(body);
|
||||
});
|
||||
});
|
||||
|
||||
*/
|
||||
|
||||
let stream = require('stream');
|
||||
const util = require('util');
|
||||
|
||||
if (!stream.Transform) {
|
||||
stream = require('readable-stream');
|
||||
}
|
||||
|
||||
function ReadToEnd(opts) {
|
||||
if (!(this instanceof ReadToEnd)) {
|
||||
return new ReadToEnd(opts);
|
||||
}
|
||||
|
||||
stream.Transform.call(this, opts);
|
||||
|
||||
this._rte_encoding = opts.encoding || 'utf8';
|
||||
|
||||
this._buff = '';
|
||||
}
|
||||
|
||||
module.exports = ReadToEnd;
|
||||
util.inherits(ReadToEnd, stream.Transform);
|
||||
|
||||
ReadToEnd.prototype._transform = function(chunk, encoding, done) {
|
||||
this._buff += chunk.toString(this._rte_encoding);
|
||||
this.push(chunk);
|
||||
done();
|
||||
};
|
||||
|
||||
ReadToEnd.prototype._flush = function(done) {
|
||||
this.emit('complete', undefined, this._buff);
|
||||
done();
|
||||
};
|
||||
|
||||
ReadToEnd.readToEnd = function(stream, options, cb) {
|
||||
if (!cb) {
|
||||
cb = options;
|
||||
options = {};
|
||||
}
|
||||
|
||||
const dest = new ReadToEnd(options);
|
||||
|
||||
stream.pipe(dest);
|
||||
|
||||
stream.on('error', function(err) {
|
||||
stream.unpipe(dest);
|
||||
cb(err);
|
||||
});
|
||||
|
||||
dest.on('complete', cb);
|
||||
|
||||
dest.resume();
|
||||
|
||||
return dest;
|
||||
};
|
||||
-108
@@ -1,108 +0,0 @@
|
||||
type X2jOptions = {
|
||||
preserveOrder: boolean;
|
||||
attributeNamePrefix: string;
|
||||
attributesGroupName: false | string;
|
||||
textNodeName: string;
|
||||
ignoreAttributes: boolean;
|
||||
removeNSPrefix: boolean;
|
||||
allowBooleanAttributes: boolean;
|
||||
parseTagValue: boolean;
|
||||
parseAttributeValue: boolean;
|
||||
trimValues: boolean;
|
||||
cdataPropName: false | string;
|
||||
commentPropName: false | string;
|
||||
/**
|
||||
Control how tag value should be parsed. Called only if tag value is not empty
|
||||
|
||||
@returns {undefined|null} `undefined` or `null` to set original value.
|
||||
@returns {unknown}
|
||||
1. Different value or value with different data type to set new value. <br>
|
||||
2. Same value to set parsed value if `parseTagValue: true`.
|
||||
*/
|
||||
tagValueProcessor: (tagName: string, tagValue: string, jPath: string, hasAttributes: boolean, isLeafNode: boolean) => unknown;
|
||||
attributeValueProcessor: (attrName: string, attrValue: string, jPath: string) => unknown;
|
||||
numberParseOptions: strnumOptions;
|
||||
stopNodes: string[];
|
||||
unpairedTags: string[];
|
||||
alwaysCreateTextNode: boolean;
|
||||
isArray: (tagName: string, jPath: string, isLeafNode: boolean, isAttribute: boolean) => boolean;
|
||||
processEntities: boolean;
|
||||
htmlEntities: boolean;
|
||||
ignoreDeclaration: boolean;
|
||||
ignorePiTags: boolean;
|
||||
transformTagName: ((tagName: string) => string) | false;
|
||||
transformAttributeName: ((attributeName: string) => string) | false;
|
||||
/**
|
||||
Change the tag name when a different name is returned. Skip the tag from parsed result when false is returned.
|
||||
Modify `attrs` object to control attributes for the given tag.
|
||||
|
||||
@returns {string} new tag name.
|
||||
@returns false to skip the tag
|
||||
*/
|
||||
updateTag: (tagName: string, jPath: string, attrs: {[k: string]: string}) => string | boolean;
|
||||
};
|
||||
type strnumOptions = {
|
||||
hex: boolean;
|
||||
leadingZeros: boolean,
|
||||
skipLike?: RegExp,
|
||||
eNotation?: boolean
|
||||
}
|
||||
type X2jOptionsOptional = Partial<X2jOptions>;
|
||||
type validationOptions = {
|
||||
allowBooleanAttributes: boolean;
|
||||
unpairedTags: string[];
|
||||
};
|
||||
type validationOptionsOptional = Partial<validationOptions>;
|
||||
|
||||
type XmlBuilderOptions = {
|
||||
attributeNamePrefix: string;
|
||||
attributesGroupName: false | string;
|
||||
textNodeName: string;
|
||||
ignoreAttributes: boolean;
|
||||
cdataPropName: false | string;
|
||||
commentPropName: false | string;
|
||||
format: boolean;
|
||||
indentBy: string;
|
||||
arrayNodeName: string;
|
||||
suppressEmptyNode: boolean;
|
||||
suppressUnpairedNode: boolean;
|
||||
suppressBooleanAttributes: boolean;
|
||||
preserveOrder: boolean;
|
||||
unpairedTags: string[];
|
||||
stopNodes: string[];
|
||||
tagValueProcessor: (name: string, value: unknown) => string;
|
||||
attributeValueProcessor: (name: string, value: unknown) => string;
|
||||
processEntities: boolean;
|
||||
oneListGroup: boolean;
|
||||
};
|
||||
type XmlBuilderOptionsOptional = Partial<XmlBuilderOptions>;
|
||||
|
||||
type ESchema = string | object | Array<string|object>;
|
||||
|
||||
type ValidationError = {
|
||||
err: {
|
||||
code: string;
|
||||
msg: string,
|
||||
line: number,
|
||||
col: number
|
||||
};
|
||||
};
|
||||
|
||||
export class XMLParser {
|
||||
constructor(options?: X2jOptionsOptional);
|
||||
parse(xmlData: string | Buffer ,validationOptions?: validationOptionsOptional | boolean): any;
|
||||
/**
|
||||
* Add Entity which is not by default supported by this library
|
||||
* @param entityIndentifier {string} Eg: 'ent' for &ent;
|
||||
* @param entityValue {string} Eg: '\r'
|
||||
*/
|
||||
addEntity(entityIndentifier: string, entityValue: string): void;
|
||||
}
|
||||
|
||||
export class XMLValidator{
|
||||
static validate( xmlData: string, options?: validationOptionsOptional): true | ValidationError;
|
||||
}
|
||||
export class XMLBuilder {
|
||||
constructor(options?: XmlBuilderOptionsOptional);
|
||||
build(jObj: any): any;
|
||||
}
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const validator = require('./validator');
|
||||
const XMLParser = require('./xmlparser/XMLParser');
|
||||
const XMLBuilder = require('./xmlbuilder/json2xml');
|
||||
|
||||
module.exports = {
|
||||
XMLParser: XMLParser,
|
||||
XMLValidator: validator,
|
||||
XMLBuilder: XMLBuilder
|
||||
}
|
||||
-72
@@ -1,72 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const nameStartChar = ':A-Za-z_\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD';
|
||||
const nameChar = nameStartChar + '\\-.\\d\\u00B7\\u0300-\\u036F\\u203F-\\u2040';
|
||||
const nameRegexp = '[' + nameStartChar + '][' + nameChar + ']*'
|
||||
const regexName = new RegExp('^' + nameRegexp + '$');
|
||||
|
||||
const getAllMatches = function(string, regex) {
|
||||
const matches = [];
|
||||
let match = regex.exec(string);
|
||||
while (match) {
|
||||
const allmatches = [];
|
||||
allmatches.startIndex = regex.lastIndex - match[0].length;
|
||||
const len = match.length;
|
||||
for (let index = 0; index < len; index++) {
|
||||
allmatches.push(match[index]);
|
||||
}
|
||||
matches.push(allmatches);
|
||||
match = regex.exec(string);
|
||||
}
|
||||
return matches;
|
||||
};
|
||||
|
||||
const isName = function(string) {
|
||||
const match = regexName.exec(string);
|
||||
return !(match === null || typeof match === 'undefined');
|
||||
};
|
||||
|
||||
exports.isExist = function(v) {
|
||||
return typeof v !== 'undefined';
|
||||
};
|
||||
|
||||
exports.isEmptyObject = function(obj) {
|
||||
return Object.keys(obj).length === 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* Copy all the properties of a into b.
|
||||
* @param {*} target
|
||||
* @param {*} a
|
||||
*/
|
||||
exports.merge = function(target, a, arrayMode) {
|
||||
if (a) {
|
||||
const keys = Object.keys(a); // will return an array of own properties
|
||||
const len = keys.length; //don't make it inline
|
||||
for (let i = 0; i < len; i++) {
|
||||
if (arrayMode === 'strict') {
|
||||
target[keys[i]] = [ a[keys[i]] ];
|
||||
} else {
|
||||
target[keys[i]] = a[keys[i]];
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
/* exports.merge =function (b,a){
|
||||
return Object.assign(b,a);
|
||||
} */
|
||||
|
||||
exports.getValue = function(v) {
|
||||
if (exports.isExist(v)) {
|
||||
return v;
|
||||
} else {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
// const fakeCall = function(a) {return a;};
|
||||
// const fakeCallNoReturn = function() {};
|
||||
|
||||
exports.isName = isName;
|
||||
exports.getAllMatches = getAllMatches;
|
||||
exports.nameRegexp = nameRegexp;
|
||||
Generated
Vendored
-423
@@ -1,423 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const util = require('./util');
|
||||
|
||||
const defaultOptions = {
|
||||
allowBooleanAttributes: false, //A tag can have attributes without any value
|
||||
unpairedTags: []
|
||||
};
|
||||
|
||||
//const tagsPattern = new RegExp("<\\/?([\\w:\\-_\.]+)\\s*\/?>","g");
|
||||
exports.validate = function (xmlData, options) {
|
||||
options = Object.assign({}, defaultOptions, options);
|
||||
|
||||
//xmlData = xmlData.replace(/(\r\n|\n|\r)/gm,"");//make it single line
|
||||
//xmlData = xmlData.replace(/(^\s*<\?xml.*?\?>)/g,"");//Remove XML starting tag
|
||||
//xmlData = xmlData.replace(/(<!DOCTYPE[\s\w\"\.\/\-\:]+(\[.*\])*\s*>)/g,"");//Remove DOCTYPE
|
||||
const tags = [];
|
||||
let tagFound = false;
|
||||
|
||||
//indicates that the root tag has been closed (aka. depth 0 has been reached)
|
||||
let reachedRoot = false;
|
||||
|
||||
if (xmlData[0] === '\ufeff') {
|
||||
// check for byte order mark (BOM)
|
||||
xmlData = xmlData.substr(1);
|
||||
}
|
||||
|
||||
for (let i = 0; i < xmlData.length; i++) {
|
||||
|
||||
if (xmlData[i] === '<' && xmlData[i+1] === '?') {
|
||||
i+=2;
|
||||
i = readPI(xmlData,i);
|
||||
if (i.err) return i;
|
||||
}else if (xmlData[i] === '<') {
|
||||
//starting of tag
|
||||
//read until you reach to '>' avoiding any '>' in attribute value
|
||||
let tagStartPos = i;
|
||||
i++;
|
||||
|
||||
if (xmlData[i] === '!') {
|
||||
i = readCommentAndCDATA(xmlData, i);
|
||||
continue;
|
||||
} else {
|
||||
let closingTag = false;
|
||||
if (xmlData[i] === '/') {
|
||||
//closing tag
|
||||
closingTag = true;
|
||||
i++;
|
||||
}
|
||||
//read tagname
|
||||
let tagName = '';
|
||||
for (; i < xmlData.length &&
|
||||
xmlData[i] !== '>' &&
|
||||
xmlData[i] !== ' ' &&
|
||||
xmlData[i] !== '\t' &&
|
||||
xmlData[i] !== '\n' &&
|
||||
xmlData[i] !== '\r'; i++
|
||||
) {
|
||||
tagName += xmlData[i];
|
||||
}
|
||||
tagName = tagName.trim();
|
||||
//console.log(tagName);
|
||||
|
||||
if (tagName[tagName.length - 1] === '/') {
|
||||
//self closing tag without attributes
|
||||
tagName = tagName.substring(0, tagName.length - 1);
|
||||
//continue;
|
||||
i--;
|
||||
}
|
||||
if (!validateTagName(tagName)) {
|
||||
let msg;
|
||||
if (tagName.trim().length === 0) {
|
||||
msg = "Invalid space after '<'.";
|
||||
} else {
|
||||
msg = "Tag '"+tagName+"' is an invalid name.";
|
||||
}
|
||||
return getErrorObject('InvalidTag', msg, getLineNumberForPosition(xmlData, i));
|
||||
}
|
||||
|
||||
const result = readAttributeStr(xmlData, i);
|
||||
if (result === false) {
|
||||
return getErrorObject('InvalidAttr', "Attributes for '"+tagName+"' have open quote.", getLineNumberForPosition(xmlData, i));
|
||||
}
|
||||
let attrStr = result.value;
|
||||
i = result.index;
|
||||
|
||||
if (attrStr[attrStr.length - 1] === '/') {
|
||||
//self closing tag
|
||||
const attrStrStart = i - attrStr.length;
|
||||
attrStr = attrStr.substring(0, attrStr.length - 1);
|
||||
const isValid = validateAttributeString(attrStr, options);
|
||||
if (isValid === true) {
|
||||
tagFound = true;
|
||||
//continue; //text may presents after self closing tag
|
||||
} else {
|
||||
//the result from the nested function returns the position of the error within the attribute
|
||||
//in order to get the 'true' error line, we need to calculate the position where the attribute begins (i - attrStr.length) and then add the position within the attribute
|
||||
//this gives us the absolute index in the entire xml, which we can use to find the line at last
|
||||
return getErrorObject(isValid.err.code, isValid.err.msg, getLineNumberForPosition(xmlData, attrStrStart + isValid.err.line));
|
||||
}
|
||||
} else if (closingTag) {
|
||||
if (!result.tagClosed) {
|
||||
return getErrorObject('InvalidTag', "Closing tag '"+tagName+"' doesn't have proper closing.", getLineNumberForPosition(xmlData, i));
|
||||
} else if (attrStr.trim().length > 0) {
|
||||
return getErrorObject('InvalidTag', "Closing tag '"+tagName+"' can't have attributes or invalid starting.", getLineNumberForPosition(xmlData, tagStartPos));
|
||||
} else {
|
||||
const otg = tags.pop();
|
||||
if (tagName !== otg.tagName) {
|
||||
let openPos = getLineNumberForPosition(xmlData, otg.tagStartPos);
|
||||
return getErrorObject('InvalidTag',
|
||||
"Expected closing tag '"+otg.tagName+"' (opened in line "+openPos.line+", col "+openPos.col+") instead of closing tag '"+tagName+"'.",
|
||||
getLineNumberForPosition(xmlData, tagStartPos));
|
||||
}
|
||||
|
||||
//when there are no more tags, we reached the root level.
|
||||
if (tags.length == 0) {
|
||||
reachedRoot = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const isValid = validateAttributeString(attrStr, options);
|
||||
if (isValid !== true) {
|
||||
//the result from the nested function returns the position of the error within the attribute
|
||||
//in order to get the 'true' error line, we need to calculate the position where the attribute begins (i - attrStr.length) and then add the position within the attribute
|
||||
//this gives us the absolute index in the entire xml, which we can use to find the line at last
|
||||
return getErrorObject(isValid.err.code, isValid.err.msg, getLineNumberForPosition(xmlData, i - attrStr.length + isValid.err.line));
|
||||
}
|
||||
|
||||
//if the root level has been reached before ...
|
||||
if (reachedRoot === true) {
|
||||
return getErrorObject('InvalidXml', 'Multiple possible root nodes found.', getLineNumberForPosition(xmlData, i));
|
||||
} else if(options.unpairedTags.indexOf(tagName) !== -1){
|
||||
//don't push into stack
|
||||
} else {
|
||||
tags.push({tagName, tagStartPos});
|
||||
}
|
||||
tagFound = true;
|
||||
}
|
||||
|
||||
//skip tag text value
|
||||
//It may include comments and CDATA value
|
||||
for (i++; i < xmlData.length; i++) {
|
||||
if (xmlData[i] === '<') {
|
||||
if (xmlData[i + 1] === '!') {
|
||||
//comment or CADATA
|
||||
i++;
|
||||
i = readCommentAndCDATA(xmlData, i);
|
||||
continue;
|
||||
} else if (xmlData[i+1] === '?') {
|
||||
i = readPI(xmlData, ++i);
|
||||
if (i.err) return i;
|
||||
} else{
|
||||
break;
|
||||
}
|
||||
} else if (xmlData[i] === '&') {
|
||||
const afterAmp = validateAmpersand(xmlData, i);
|
||||
if (afterAmp == -1)
|
||||
return getErrorObject('InvalidChar', "char '&' is not expected.", getLineNumberForPosition(xmlData, i));
|
||||
i = afterAmp;
|
||||
}else{
|
||||
if (reachedRoot === true && !isWhiteSpace(xmlData[i])) {
|
||||
return getErrorObject('InvalidXml', "Extra text at the end", getLineNumberForPosition(xmlData, i));
|
||||
}
|
||||
}
|
||||
} //end of reading tag text value
|
||||
if (xmlData[i] === '<') {
|
||||
i--;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if ( isWhiteSpace(xmlData[i])) {
|
||||
continue;
|
||||
}
|
||||
return getErrorObject('InvalidChar', "char '"+xmlData[i]+"' is not expected.", getLineNumberForPosition(xmlData, i));
|
||||
}
|
||||
}
|
||||
|
||||
if (!tagFound) {
|
||||
return getErrorObject('InvalidXml', 'Start tag expected.', 1);
|
||||
}else if (tags.length == 1) {
|
||||
return getErrorObject('InvalidTag', "Unclosed tag '"+tags[0].tagName+"'.", getLineNumberForPosition(xmlData, tags[0].tagStartPos));
|
||||
}else if (tags.length > 0) {
|
||||
return getErrorObject('InvalidXml', "Invalid '"+
|
||||
JSON.stringify(tags.map(t => t.tagName), null, 4).replace(/\r?\n/g, '')+
|
||||
"' found.", {line: 1, col: 1});
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
function isWhiteSpace(char){
|
||||
return char === ' ' || char === '\t' || char === '\n' || char === '\r';
|
||||
}
|
||||
/**
|
||||
* Read Processing insstructions and skip
|
||||
* @param {*} xmlData
|
||||
* @param {*} i
|
||||
*/
|
||||
function readPI(xmlData, i) {
|
||||
const start = i;
|
||||
for (; i < xmlData.length; i++) {
|
||||
if (xmlData[i] == '?' || xmlData[i] == ' ') {
|
||||
//tagname
|
||||
const tagname = xmlData.substr(start, i - start);
|
||||
if (i > 5 && tagname === 'xml') {
|
||||
return getErrorObject('InvalidXml', 'XML declaration allowed only at the start of the document.', getLineNumberForPosition(xmlData, i));
|
||||
} else if (xmlData[i] == '?' && xmlData[i + 1] == '>') {
|
||||
//check if valid attribut string
|
||||
i++;
|
||||
break;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
function readCommentAndCDATA(xmlData, i) {
|
||||
if (xmlData.length > i + 5 && xmlData[i + 1] === '-' && xmlData[i + 2] === '-') {
|
||||
//comment
|
||||
for (i += 3; i < xmlData.length; i++) {
|
||||
if (xmlData[i] === '-' && xmlData[i + 1] === '-' && xmlData[i + 2] === '>') {
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (
|
||||
xmlData.length > i + 8 &&
|
||||
xmlData[i + 1] === 'D' &&
|
||||
xmlData[i + 2] === 'O' &&
|
||||
xmlData[i + 3] === 'C' &&
|
||||
xmlData[i + 4] === 'T' &&
|
||||
xmlData[i + 5] === 'Y' &&
|
||||
xmlData[i + 6] === 'P' &&
|
||||
xmlData[i + 7] === 'E'
|
||||
) {
|
||||
let angleBracketsCount = 1;
|
||||
for (i += 8; i < xmlData.length; i++) {
|
||||
if (xmlData[i] === '<') {
|
||||
angleBracketsCount++;
|
||||
} else if (xmlData[i] === '>') {
|
||||
angleBracketsCount--;
|
||||
if (angleBracketsCount === 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (
|
||||
xmlData.length > i + 9 &&
|
||||
xmlData[i + 1] === '[' &&
|
||||
xmlData[i + 2] === 'C' &&
|
||||
xmlData[i + 3] === 'D' &&
|
||||
xmlData[i + 4] === 'A' &&
|
||||
xmlData[i + 5] === 'T' &&
|
||||
xmlData[i + 6] === 'A' &&
|
||||
xmlData[i + 7] === '['
|
||||
) {
|
||||
for (i += 8; i < xmlData.length; i++) {
|
||||
if (xmlData[i] === ']' && xmlData[i + 1] === ']' && xmlData[i + 2] === '>') {
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return i;
|
||||
}
|
||||
|
||||
const doubleQuote = '"';
|
||||
const singleQuote = "'";
|
||||
|
||||
/**
|
||||
* Keep reading xmlData until '<' is found outside the attribute value.
|
||||
* @param {string} xmlData
|
||||
* @param {number} i
|
||||
*/
|
||||
function readAttributeStr(xmlData, i) {
|
||||
let attrStr = '';
|
||||
let startChar = '';
|
||||
let tagClosed = false;
|
||||
for (; i < xmlData.length; i++) {
|
||||
if (xmlData[i] === doubleQuote || xmlData[i] === singleQuote) {
|
||||
if (startChar === '') {
|
||||
startChar = xmlData[i];
|
||||
} else if (startChar !== xmlData[i]) {
|
||||
//if vaue is enclosed with double quote then single quotes are allowed inside the value and vice versa
|
||||
} else {
|
||||
startChar = '';
|
||||
}
|
||||
} else if (xmlData[i] === '>') {
|
||||
if (startChar === '') {
|
||||
tagClosed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
attrStr += xmlData[i];
|
||||
}
|
||||
if (startChar !== '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
return {
|
||||
value: attrStr,
|
||||
index: i,
|
||||
tagClosed: tagClosed
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Select all the attributes whether valid or invalid.
|
||||
*/
|
||||
const validAttrStrRegxp = new RegExp('(\\s*)([^\\s=]+)(\\s*=)?(\\s*([\'"])(([\\s\\S])*?)\\5)?', 'g');
|
||||
|
||||
//attr, ="sd", a="amit's", a="sd"b="saf", ab cd=""
|
||||
|
||||
function validateAttributeString(attrStr, options) {
|
||||
//console.log("start:"+attrStr+":end");
|
||||
|
||||
//if(attrStr.trim().length === 0) return true; //empty string
|
||||
|
||||
const matches = util.getAllMatches(attrStr, validAttrStrRegxp);
|
||||
const attrNames = {};
|
||||
|
||||
for (let i = 0; i < matches.length; i++) {
|
||||
if (matches[i][1].length === 0) {
|
||||
//nospace before attribute name: a="sd"b="saf"
|
||||
return getErrorObject('InvalidAttr', "Attribute '"+matches[i][2]+"' has no space in starting.", getPositionFromMatch(matches[i]))
|
||||
} else if (matches[i][3] !== undefined && matches[i][4] === undefined) {
|
||||
return getErrorObject('InvalidAttr', "Attribute '"+matches[i][2]+"' is without value.", getPositionFromMatch(matches[i]));
|
||||
} else if (matches[i][3] === undefined && !options.allowBooleanAttributes) {
|
||||
//independent attribute: ab
|
||||
return getErrorObject('InvalidAttr', "boolean attribute '"+matches[i][2]+"' is not allowed.", getPositionFromMatch(matches[i]));
|
||||
}
|
||||
/* else if(matches[i][6] === undefined){//attribute without value: ab=
|
||||
return { err: { code:"InvalidAttr",msg:"attribute " + matches[i][2] + " has no value assigned."}};
|
||||
} */
|
||||
const attrName = matches[i][2];
|
||||
if (!validateAttrName(attrName)) {
|
||||
return getErrorObject('InvalidAttr', "Attribute '"+attrName+"' is an invalid name.", getPositionFromMatch(matches[i]));
|
||||
}
|
||||
if (!attrNames.hasOwnProperty(attrName)) {
|
||||
//check for duplicate attribute.
|
||||
attrNames[attrName] = 1;
|
||||
} else {
|
||||
return getErrorObject('InvalidAttr', "Attribute '"+attrName+"' is repeated.", getPositionFromMatch(matches[i]));
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
function validateNumberAmpersand(xmlData, i) {
|
||||
let re = /\d/;
|
||||
if (xmlData[i] === 'x') {
|
||||
i++;
|
||||
re = /[\da-fA-F]/;
|
||||
}
|
||||
for (; i < xmlData.length; i++) {
|
||||
if (xmlData[i] === ';')
|
||||
return i;
|
||||
if (!xmlData[i].match(re))
|
||||
break;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function validateAmpersand(xmlData, i) {
|
||||
// https://www.w3.org/TR/xml/#dt-charref
|
||||
i++;
|
||||
if (xmlData[i] === ';')
|
||||
return -1;
|
||||
if (xmlData[i] === '#') {
|
||||
i++;
|
||||
return validateNumberAmpersand(xmlData, i);
|
||||
}
|
||||
let count = 0;
|
||||
for (; i < xmlData.length; i++, count++) {
|
||||
if (xmlData[i].match(/\w/) && count < 20)
|
||||
continue;
|
||||
if (xmlData[i] === ';')
|
||||
break;
|
||||
return -1;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
function getErrorObject(code, message, lineNumber) {
|
||||
return {
|
||||
err: {
|
||||
code: code,
|
||||
msg: message,
|
||||
line: lineNumber.line || lineNumber,
|
||||
col: lineNumber.col,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function validateAttrName(attrName) {
|
||||
return util.isName(attrName);
|
||||
}
|
||||
|
||||
// const startsWithXML = /^xml/i;
|
||||
|
||||
function validateTagName(tagname) {
|
||||
return util.isName(tagname) /* && !tagname.match(startsWithXML) */;
|
||||
}
|
||||
|
||||
//this function returns the line number for the character at the given index
|
||||
function getLineNumberForPosition(xmlData, index) {
|
||||
const lines = xmlData.substring(0, index).split(/\r?\n/);
|
||||
return {
|
||||
line: lines.length,
|
||||
|
||||
// column number is last line's length + 1, because column numbering starts at 1:
|
||||
col: lines[lines.length - 1].length + 1
|
||||
};
|
||||
}
|
||||
|
||||
//this function returns the position of the first character of match within attrStr
|
||||
function getPositionFromMatch(match) {
|
||||
return match.startIndex + match[1].length;
|
||||
}
|
||||
Generated
Vendored
-270
@@ -1,270 +0,0 @@
|
||||
'use strict';
|
||||
//parse Empty Node as self closing node
|
||||
const buildFromOrderedJs = require('./orderedJs2Xml');
|
||||
|
||||
const defaultOptions = {
|
||||
attributeNamePrefix: '@_',
|
||||
attributesGroupName: false,
|
||||
textNodeName: '#text',
|
||||
ignoreAttributes: true,
|
||||
cdataPropName: false,
|
||||
format: false,
|
||||
indentBy: ' ',
|
||||
suppressEmptyNode: false,
|
||||
suppressUnpairedNode: true,
|
||||
suppressBooleanAttributes: true,
|
||||
tagValueProcessor: function(key, a) {
|
||||
return a;
|
||||
},
|
||||
attributeValueProcessor: function(attrName, a) {
|
||||
return a;
|
||||
},
|
||||
preserveOrder: false,
|
||||
commentPropName: false,
|
||||
unpairedTags: [],
|
||||
entities: [
|
||||
{ regex: new RegExp("&", "g"), val: "&" },//it must be on top
|
||||
{ regex: new RegExp(">", "g"), val: ">" },
|
||||
{ regex: new RegExp("<", "g"), val: "<" },
|
||||
{ regex: new RegExp("\'", "g"), val: "'" },
|
||||
{ regex: new RegExp("\"", "g"), val: """ }
|
||||
],
|
||||
processEntities: true,
|
||||
stopNodes: [],
|
||||
// transformTagName: false,
|
||||
// transformAttributeName: false,
|
||||
oneListGroup: false
|
||||
};
|
||||
|
||||
function Builder(options) {
|
||||
this.options = Object.assign({}, defaultOptions, options);
|
||||
if (this.options.ignoreAttributes || this.options.attributesGroupName) {
|
||||
this.isAttribute = function(/*a*/) {
|
||||
return false;
|
||||
};
|
||||
} else {
|
||||
this.attrPrefixLen = this.options.attributeNamePrefix.length;
|
||||
this.isAttribute = isAttribute;
|
||||
}
|
||||
|
||||
this.processTextOrObjNode = processTextOrObjNode
|
||||
|
||||
if (this.options.format) {
|
||||
this.indentate = indentate;
|
||||
this.tagEndChar = '>\n';
|
||||
this.newLine = '\n';
|
||||
} else {
|
||||
this.indentate = function() {
|
||||
return '';
|
||||
};
|
||||
this.tagEndChar = '>';
|
||||
this.newLine = '';
|
||||
}
|
||||
}
|
||||
|
||||
Builder.prototype.build = function(jObj) {
|
||||
if(this.options.preserveOrder){
|
||||
return buildFromOrderedJs(jObj, this.options);
|
||||
}else {
|
||||
if(Array.isArray(jObj) && this.options.arrayNodeName && this.options.arrayNodeName.length > 1){
|
||||
jObj = {
|
||||
[this.options.arrayNodeName] : jObj
|
||||
}
|
||||
}
|
||||
return this.j2x(jObj, 0).val;
|
||||
}
|
||||
};
|
||||
|
||||
Builder.prototype.j2x = function(jObj, level) {
|
||||
let attrStr = '';
|
||||
let val = '';
|
||||
for (let key in jObj) {
|
||||
if(!Object.prototype.hasOwnProperty.call(jObj, key)) continue;
|
||||
if (typeof jObj[key] === 'undefined') {
|
||||
// supress undefined node only if it is not an attribute
|
||||
if (this.isAttribute(key)) {
|
||||
val += '';
|
||||
}
|
||||
} else if (jObj[key] === null) {
|
||||
// null attribute should be ignored by the attribute list, but should not cause the tag closing
|
||||
if (this.isAttribute(key)) {
|
||||
val += '';
|
||||
} else if (key[0] === '?') {
|
||||
val += this.indentate(level) + '<' + key + '?' + this.tagEndChar;
|
||||
} else {
|
||||
val += this.indentate(level) + '<' + key + '/' + this.tagEndChar;
|
||||
}
|
||||
// val += this.indentate(level) + '<' + key + '/' + this.tagEndChar;
|
||||
} else if (jObj[key] instanceof Date) {
|
||||
val += this.buildTextValNode(jObj[key], key, '', level);
|
||||
} else if (typeof jObj[key] !== 'object') {
|
||||
//premitive type
|
||||
const attr = this.isAttribute(key);
|
||||
if (attr) {
|
||||
attrStr += this.buildAttrPairStr(attr, '' + jObj[key]);
|
||||
}else {
|
||||
//tag value
|
||||
if (key === this.options.textNodeName) {
|
||||
let newval = this.options.tagValueProcessor(key, '' + jObj[key]);
|
||||
val += this.replaceEntitiesValue(newval);
|
||||
} else {
|
||||
val += this.buildTextValNode(jObj[key], key, '', level);
|
||||
}
|
||||
}
|
||||
} else if (Array.isArray(jObj[key])) {
|
||||
//repeated nodes
|
||||
const arrLen = jObj[key].length;
|
||||
let listTagVal = "";
|
||||
for (let j = 0; j < arrLen; j++) {
|
||||
const item = jObj[key][j];
|
||||
if (typeof item === 'undefined') {
|
||||
// supress undefined node
|
||||
} else if (item === null) {
|
||||
if(key[0] === "?") val += this.indentate(level) + '<' + key + '?' + this.tagEndChar;
|
||||
else val += this.indentate(level) + '<' + key + '/' + this.tagEndChar;
|
||||
// val += this.indentate(level) + '<' + key + '/' + this.tagEndChar;
|
||||
} else if (typeof item === 'object') {
|
||||
if(this.options.oneListGroup ){
|
||||
listTagVal += this.j2x(item, level + 1).val;
|
||||
}else{
|
||||
listTagVal += this.processTextOrObjNode(item, key, level)
|
||||
}
|
||||
} else {
|
||||
listTagVal += this.buildTextValNode(item, key, '', level);
|
||||
}
|
||||
}
|
||||
if(this.options.oneListGroup){
|
||||
listTagVal = this.buildObjectNode(listTagVal, key, '', level);
|
||||
}
|
||||
val += listTagVal;
|
||||
} else {
|
||||
//nested node
|
||||
if (this.options.attributesGroupName && key === this.options.attributesGroupName) {
|
||||
const Ks = Object.keys(jObj[key]);
|
||||
const L = Ks.length;
|
||||
for (let j = 0; j < L; j++) {
|
||||
attrStr += this.buildAttrPairStr(Ks[j], '' + jObj[key][Ks[j]]);
|
||||
}
|
||||
} else {
|
||||
val += this.processTextOrObjNode(jObj[key], key, level)
|
||||
}
|
||||
}
|
||||
}
|
||||
return {attrStr: attrStr, val: val};
|
||||
};
|
||||
|
||||
Builder.prototype.buildAttrPairStr = function(attrName, val){
|
||||
val = this.options.attributeValueProcessor(attrName, '' + val);
|
||||
val = this.replaceEntitiesValue(val);
|
||||
if (this.options.suppressBooleanAttributes && val === "true") {
|
||||
return ' ' + attrName;
|
||||
} else return ' ' + attrName + '="' + val + '"';
|
||||
}
|
||||
|
||||
function processTextOrObjNode (object, key, level) {
|
||||
const result = this.j2x(object, level + 1);
|
||||
if (object[this.options.textNodeName] !== undefined && Object.keys(object).length === 1) {
|
||||
return this.buildTextValNode(object[this.options.textNodeName], key, result.attrStr, level);
|
||||
} else {
|
||||
return this.buildObjectNode(result.val, key, result.attrStr, level);
|
||||
}
|
||||
}
|
||||
|
||||
Builder.prototype.buildObjectNode = function(val, key, attrStr, level) {
|
||||
if(val === ""){
|
||||
if(key[0] === "?") return this.indentate(level) + '<' + key + attrStr+ '?' + this.tagEndChar;
|
||||
else {
|
||||
return this.indentate(level) + '<' + key + attrStr + this.closeTag(key) + this.tagEndChar;
|
||||
}
|
||||
}else{
|
||||
|
||||
let tagEndExp = '</' + key + this.tagEndChar;
|
||||
let piClosingChar = "";
|
||||
|
||||
if(key[0] === "?") {
|
||||
piClosingChar = "?";
|
||||
tagEndExp = "";
|
||||
}
|
||||
|
||||
// attrStr is an empty string in case the attribute came as undefined or null
|
||||
if ((attrStr || attrStr === '') && val.indexOf('<') === -1) {
|
||||
return ( this.indentate(level) + '<' + key + attrStr + piClosingChar + '>' + val + tagEndExp );
|
||||
} else if (this.options.commentPropName !== false && key === this.options.commentPropName && piClosingChar.length === 0) {
|
||||
return this.indentate(level) + `<!--${val}-->` + this.newLine;
|
||||
}else {
|
||||
return (
|
||||
this.indentate(level) + '<' + key + attrStr + piClosingChar + this.tagEndChar +
|
||||
val +
|
||||
this.indentate(level) + tagEndExp );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Builder.prototype.closeTag = function(key){
|
||||
let closeTag = "";
|
||||
if(this.options.unpairedTags.indexOf(key) !== -1){ //unpaired
|
||||
if(!this.options.suppressUnpairedNode) closeTag = "/"
|
||||
}else if(this.options.suppressEmptyNode){ //empty
|
||||
closeTag = "/";
|
||||
}else{
|
||||
closeTag = `></${key}`
|
||||
}
|
||||
return closeTag;
|
||||
}
|
||||
|
||||
function buildEmptyObjNode(val, key, attrStr, level) {
|
||||
if (val !== '') {
|
||||
return this.buildObjectNode(val, key, attrStr, level);
|
||||
} else {
|
||||
if(key[0] === "?") return this.indentate(level) + '<' + key + attrStr+ '?' + this.tagEndChar;
|
||||
else {
|
||||
return this.indentate(level) + '<' + key + attrStr + '/' + this.tagEndChar;
|
||||
// return this.buildTagStr(level,key, attrStr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Builder.prototype.buildTextValNode = function(val, key, attrStr, level) {
|
||||
if (this.options.cdataPropName !== false && key === this.options.cdataPropName) {
|
||||
return this.indentate(level) + `<![CDATA[${val}]]>` + this.newLine;
|
||||
}else if (this.options.commentPropName !== false && key === this.options.commentPropName) {
|
||||
return this.indentate(level) + `<!--${val}-->` + this.newLine;
|
||||
}else if(key[0] === "?") {//PI tag
|
||||
return this.indentate(level) + '<' + key + attrStr+ '?' + this.tagEndChar;
|
||||
}else{
|
||||
let textValue = this.options.tagValueProcessor(key, val);
|
||||
textValue = this.replaceEntitiesValue(textValue);
|
||||
|
||||
if( textValue === ''){
|
||||
return this.indentate(level) + '<' + key + attrStr + this.closeTag(key) + this.tagEndChar;
|
||||
}else{
|
||||
return this.indentate(level) + '<' + key + attrStr + '>' +
|
||||
textValue +
|
||||
'</' + key + this.tagEndChar;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Builder.prototype.replaceEntitiesValue = function(textValue){
|
||||
if(textValue && textValue.length > 0 && this.options.processEntities){
|
||||
for (let i=0; i<this.options.entities.length; i++) {
|
||||
const entity = this.options.entities[i];
|
||||
textValue = textValue.replace(entity.regex, entity.val);
|
||||
}
|
||||
}
|
||||
return textValue;
|
||||
}
|
||||
|
||||
function indentate(level) {
|
||||
return this.options.indentBy.repeat(level);
|
||||
}
|
||||
|
||||
function isAttribute(name /*, options*/) {
|
||||
if (name.startsWith(this.options.attributeNamePrefix) && name !== this.options.textNodeName) {
|
||||
return name.substr(this.attrPrefixLen);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Builder;
|
||||
Generated
Vendored
-135
@@ -1,135 +0,0 @@
|
||||
const EOL = "\n";
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {array} jArray
|
||||
* @param {any} options
|
||||
* @returns
|
||||
*/
|
||||
function toXml(jArray, options) {
|
||||
let indentation = "";
|
||||
if (options.format && options.indentBy.length > 0) {
|
||||
indentation = EOL;
|
||||
}
|
||||
return arrToStr(jArray, options, "", indentation);
|
||||
}
|
||||
|
||||
function arrToStr(arr, options, jPath, indentation) {
|
||||
let xmlStr = "";
|
||||
let isPreviousElementTag = false;
|
||||
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
const tagObj = arr[i];
|
||||
const tagName = propName(tagObj);
|
||||
if(tagName === undefined) continue;
|
||||
|
||||
let newJPath = "";
|
||||
if (jPath.length === 0) newJPath = tagName
|
||||
else newJPath = `${jPath}.${tagName}`;
|
||||
|
||||
if (tagName === options.textNodeName) {
|
||||
let tagText = tagObj[tagName];
|
||||
if (!isStopNode(newJPath, options)) {
|
||||
tagText = options.tagValueProcessor(tagName, tagText);
|
||||
tagText = replaceEntitiesValue(tagText, options);
|
||||
}
|
||||
if (isPreviousElementTag) {
|
||||
xmlStr += indentation;
|
||||
}
|
||||
xmlStr += tagText;
|
||||
isPreviousElementTag = false;
|
||||
continue;
|
||||
} else if (tagName === options.cdataPropName) {
|
||||
if (isPreviousElementTag) {
|
||||
xmlStr += indentation;
|
||||
}
|
||||
xmlStr += `<![CDATA[${tagObj[tagName][0][options.textNodeName]}]]>`;
|
||||
isPreviousElementTag = false;
|
||||
continue;
|
||||
} else if (tagName === options.commentPropName) {
|
||||
xmlStr += indentation + `<!--${tagObj[tagName][0][options.textNodeName]}-->`;
|
||||
isPreviousElementTag = true;
|
||||
continue;
|
||||
} else if (tagName[0] === "?") {
|
||||
const attStr = attr_to_str(tagObj[":@"], options);
|
||||
const tempInd = tagName === "?xml" ? "" : indentation;
|
||||
let piTextNodeName = tagObj[tagName][0][options.textNodeName];
|
||||
piTextNodeName = piTextNodeName.length !== 0 ? " " + piTextNodeName : ""; //remove extra spacing
|
||||
xmlStr += tempInd + `<${tagName}${piTextNodeName}${attStr}?>`;
|
||||
isPreviousElementTag = true;
|
||||
continue;
|
||||
}
|
||||
let newIdentation = indentation;
|
||||
if (newIdentation !== "") {
|
||||
newIdentation += options.indentBy;
|
||||
}
|
||||
const attStr = attr_to_str(tagObj[":@"], options);
|
||||
const tagStart = indentation + `<${tagName}${attStr}`;
|
||||
const tagValue = arrToStr(tagObj[tagName], options, newJPath, newIdentation);
|
||||
if (options.unpairedTags.indexOf(tagName) !== -1) {
|
||||
if (options.suppressUnpairedNode) xmlStr += tagStart + ">";
|
||||
else xmlStr += tagStart + "/>";
|
||||
} else if ((!tagValue || tagValue.length === 0) && options.suppressEmptyNode) {
|
||||
xmlStr += tagStart + "/>";
|
||||
} else if (tagValue && tagValue.endsWith(">")) {
|
||||
xmlStr += tagStart + `>${tagValue}${indentation}</${tagName}>`;
|
||||
} else {
|
||||
xmlStr += tagStart + ">";
|
||||
if (tagValue && indentation !== "" && (tagValue.includes("/>") || tagValue.includes("</"))) {
|
||||
xmlStr += indentation + options.indentBy + tagValue + indentation;
|
||||
} else {
|
||||
xmlStr += tagValue;
|
||||
}
|
||||
xmlStr += `</${tagName}>`;
|
||||
}
|
||||
isPreviousElementTag = true;
|
||||
}
|
||||
|
||||
return xmlStr;
|
||||
}
|
||||
|
||||
function propName(obj) {
|
||||
const keys = Object.keys(obj);
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
const key = keys[i];
|
||||
if(!obj.hasOwnProperty(key)) continue;
|
||||
if (key !== ":@") return key;
|
||||
}
|
||||
}
|
||||
|
||||
function attr_to_str(attrMap, options) {
|
||||
let attrStr = "";
|
||||
if (attrMap && !options.ignoreAttributes) {
|
||||
for (let attr in attrMap) {
|
||||
if(!attrMap.hasOwnProperty(attr)) continue;
|
||||
let attrVal = options.attributeValueProcessor(attr, attrMap[attr]);
|
||||
attrVal = replaceEntitiesValue(attrVal, options);
|
||||
if (attrVal === true && options.suppressBooleanAttributes) {
|
||||
attrStr += ` ${attr.substr(options.attributeNamePrefix.length)}`;
|
||||
} else {
|
||||
attrStr += ` ${attr.substr(options.attributeNamePrefix.length)}="${attrVal}"`;
|
||||
}
|
||||
}
|
||||
}
|
||||
return attrStr;
|
||||
}
|
||||
|
||||
function isStopNode(jPath, options) {
|
||||
jPath = jPath.substr(0, jPath.length - options.textNodeName.length - 1);
|
||||
let tagName = jPath.substr(jPath.lastIndexOf(".") + 1);
|
||||
for (let index in options.stopNodes) {
|
||||
if (options.stopNodes[index] === jPath || options.stopNodes[index] === "*." + tagName) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function replaceEntitiesValue(textValue, options) {
|
||||
if (textValue && textValue.length > 0 && options.processEntities) {
|
||||
for (let i = 0; i < options.entities.length; i++) {
|
||||
const entity = options.entities[i];
|
||||
textValue = textValue.replace(entity.regex, entity.val);
|
||||
}
|
||||
}
|
||||
return textValue;
|
||||
}
|
||||
module.exports = toXml;
|
||||
Generated
Vendored
-152
@@ -1,152 +0,0 @@
|
||||
const util = require('../util');
|
||||
|
||||
//TODO: handle comments
|
||||
function readDocType(xmlData, i){
|
||||
|
||||
const entities = {};
|
||||
if( xmlData[i + 3] === 'O' &&
|
||||
xmlData[i + 4] === 'C' &&
|
||||
xmlData[i + 5] === 'T' &&
|
||||
xmlData[i + 6] === 'Y' &&
|
||||
xmlData[i + 7] === 'P' &&
|
||||
xmlData[i + 8] === 'E')
|
||||
{
|
||||
i = i+9;
|
||||
let angleBracketsCount = 1;
|
||||
let hasBody = false, comment = false;
|
||||
let exp = "";
|
||||
for(;i<xmlData.length;i++){
|
||||
if (xmlData[i] === '<' && !comment) { //Determine the tag type
|
||||
if( hasBody && isEntity(xmlData, i)){
|
||||
i += 7;
|
||||
[entityName, val,i] = readEntityExp(xmlData,i+1);
|
||||
if(val.indexOf("&") === -1) //Parameter entities are not supported
|
||||
entities[ validateEntityName(entityName) ] = {
|
||||
regx : RegExp( `&${entityName};`,"g"),
|
||||
val: val
|
||||
};
|
||||
}
|
||||
else if( hasBody && isElement(xmlData, i)) i += 8;//Not supported
|
||||
else if( hasBody && isAttlist(xmlData, i)) i += 8;//Not supported
|
||||
else if( hasBody && isNotation(xmlData, i)) i += 9;//Not supported
|
||||
else if( isComment) comment = true;
|
||||
else throw new Error("Invalid DOCTYPE");
|
||||
|
||||
angleBracketsCount++;
|
||||
exp = "";
|
||||
} else if (xmlData[i] === '>') { //Read tag content
|
||||
if(comment){
|
||||
if( xmlData[i - 1] === "-" && xmlData[i - 2] === "-"){
|
||||
comment = false;
|
||||
angleBracketsCount--;
|
||||
}
|
||||
}else{
|
||||
angleBracketsCount--;
|
||||
}
|
||||
if (angleBracketsCount === 0) {
|
||||
break;
|
||||
}
|
||||
}else if( xmlData[i] === '['){
|
||||
hasBody = true;
|
||||
}else{
|
||||
exp += xmlData[i];
|
||||
}
|
||||
}
|
||||
if(angleBracketsCount !== 0){
|
||||
throw new Error(`Unclosed DOCTYPE`);
|
||||
}
|
||||
}else{
|
||||
throw new Error(`Invalid Tag instead of DOCTYPE`);
|
||||
}
|
||||
return {entities, i};
|
||||
}
|
||||
|
||||
function readEntityExp(xmlData,i){
|
||||
//External entities are not supported
|
||||
// <!ENTITY ext SYSTEM "http://normal-website.com" >
|
||||
|
||||
//Parameter entities are not supported
|
||||
// <!ENTITY entityname "&anotherElement;">
|
||||
|
||||
//Internal entities are supported
|
||||
// <!ENTITY entityname "replacement text">
|
||||
|
||||
//read EntityName
|
||||
let entityName = "";
|
||||
for (; i < xmlData.length && (xmlData[i] !== "'" && xmlData[i] !== '"' ); i++) {
|
||||
// if(xmlData[i] === " ") continue;
|
||||
// else
|
||||
entityName += xmlData[i];
|
||||
}
|
||||
entityName = entityName.trim();
|
||||
if(entityName.indexOf(" ") !== -1) throw new Error("External entites are not supported");
|
||||
|
||||
//read Entity Value
|
||||
const startChar = xmlData[i++];
|
||||
let val = ""
|
||||
for (; i < xmlData.length && xmlData[i] !== startChar ; i++) {
|
||||
val += xmlData[i];
|
||||
}
|
||||
return [entityName, val, i];
|
||||
}
|
||||
|
||||
function isComment(xmlData, i){
|
||||
if(xmlData[i+1] === '!' &&
|
||||
xmlData[i+2] === '-' &&
|
||||
xmlData[i+3] === '-') return true
|
||||
return false
|
||||
}
|
||||
function isEntity(xmlData, i){
|
||||
if(xmlData[i+1] === '!' &&
|
||||
xmlData[i+2] === 'E' &&
|
||||
xmlData[i+3] === 'N' &&
|
||||
xmlData[i+4] === 'T' &&
|
||||
xmlData[i+5] === 'I' &&
|
||||
xmlData[i+6] === 'T' &&
|
||||
xmlData[i+7] === 'Y') return true
|
||||
return false
|
||||
}
|
||||
function isElement(xmlData, i){
|
||||
if(xmlData[i+1] === '!' &&
|
||||
xmlData[i+2] === 'E' &&
|
||||
xmlData[i+3] === 'L' &&
|
||||
xmlData[i+4] === 'E' &&
|
||||
xmlData[i+5] === 'M' &&
|
||||
xmlData[i+6] === 'E' &&
|
||||
xmlData[i+7] === 'N' &&
|
||||
xmlData[i+8] === 'T') return true
|
||||
return false
|
||||
}
|
||||
|
||||
function isAttlist(xmlData, i){
|
||||
if(xmlData[i+1] === '!' &&
|
||||
xmlData[i+2] === 'A' &&
|
||||
xmlData[i+3] === 'T' &&
|
||||
xmlData[i+4] === 'T' &&
|
||||
xmlData[i+5] === 'L' &&
|
||||
xmlData[i+6] === 'I' &&
|
||||
xmlData[i+7] === 'S' &&
|
||||
xmlData[i+8] === 'T') return true
|
||||
return false
|
||||
}
|
||||
function isNotation(xmlData, i){
|
||||
if(xmlData[i+1] === '!' &&
|
||||
xmlData[i+2] === 'N' &&
|
||||
xmlData[i+3] === 'O' &&
|
||||
xmlData[i+4] === 'T' &&
|
||||
xmlData[i+5] === 'A' &&
|
||||
xmlData[i+6] === 'T' &&
|
||||
xmlData[i+7] === 'I' &&
|
||||
xmlData[i+8] === 'O' &&
|
||||
xmlData[i+9] === 'N') return true
|
||||
return false
|
||||
}
|
||||
|
||||
function validateEntityName(name){
|
||||
if (util.isName(name))
|
||||
return name;
|
||||
else
|
||||
throw new Error(`Invalid entity name ${name}`);
|
||||
}
|
||||
|
||||
module.exports = readDocType;
|
||||
Generated
Vendored
-48
@@ -1,48 +0,0 @@
|
||||
|
||||
const defaultOptions = {
|
||||
preserveOrder: false,
|
||||
attributeNamePrefix: '@_',
|
||||
attributesGroupName: false,
|
||||
textNodeName: '#text',
|
||||
ignoreAttributes: true,
|
||||
removeNSPrefix: false, // remove NS from tag name or attribute name if true
|
||||
allowBooleanAttributes: false, //a tag can have attributes without any value
|
||||
//ignoreRootElement : false,
|
||||
parseTagValue: true,
|
||||
parseAttributeValue: false,
|
||||
trimValues: true, //Trim string values of tag and attributes
|
||||
cdataPropName: false,
|
||||
numberParseOptions: {
|
||||
hex: true,
|
||||
leadingZeros: true,
|
||||
eNotation: true
|
||||
},
|
||||
tagValueProcessor: function(tagName, val) {
|
||||
return val;
|
||||
},
|
||||
attributeValueProcessor: function(attrName, val) {
|
||||
return val;
|
||||
},
|
||||
stopNodes: [], //nested tags will not be parsed even for errors
|
||||
alwaysCreateTextNode: false,
|
||||
isArray: () => false,
|
||||
commentPropName: false,
|
||||
unpairedTags: [],
|
||||
processEntities: true,
|
||||
htmlEntities: false,
|
||||
ignoreDeclaration: false,
|
||||
ignorePiTags: false,
|
||||
transformTagName: false,
|
||||
transformAttributeName: false,
|
||||
updateTag: function(tagName, jPath, attrs){
|
||||
return tagName
|
||||
},
|
||||
// skipEmptyListItem: false
|
||||
};
|
||||
|
||||
const buildOptions = function(options) {
|
||||
return Object.assign({}, defaultOptions, options);
|
||||
};
|
||||
|
||||
exports.buildOptions = buildOptions;
|
||||
exports.defaultOptions = defaultOptions;
|
||||
Generated
Vendored
-592
@@ -1,592 +0,0 @@
|
||||
'use strict';
|
||||
///@ts-check
|
||||
|
||||
const util = require('../util');
|
||||
const xmlNode = require('./xmlNode');
|
||||
const readDocType = require("./DocTypeReader");
|
||||
const toNumber = require("strnum");
|
||||
|
||||
const regx =
|
||||
'<((!\\[CDATA\\[([\\s\\S]*?)(]]>))|((NAME:)?(NAME))([^>]*)>|((\\/)(NAME)\\s*>))([^<]*)'
|
||||
.replace(/NAME/g, util.nameRegexp);
|
||||
|
||||
//const tagsRegx = new RegExp("<(\\/?[\\w:\\-\._]+)([^>]*)>(\\s*"+cdataRegx+")*([^<]+)?","g");
|
||||
//const tagsRegx = new RegExp("<(\\/?)((\\w*:)?([\\w:\\-\._]+))([^>]*)>([^<]*)("+cdataRegx+"([^<]*))*([^<]+)?","g");
|
||||
|
||||
class OrderedObjParser{
|
||||
constructor(options){
|
||||
this.options = options;
|
||||
this.currentNode = null;
|
||||
this.tagsNodeStack = [];
|
||||
this.docTypeEntities = {};
|
||||
this.lastEntities = {
|
||||
"apos" : { regex: /&(apos|#39|#x27);/g, val : "'"},
|
||||
"gt" : { regex: /&(gt|#62|#x3E);/g, val : ">"},
|
||||
"lt" : { regex: /&(lt|#60|#x3C);/g, val : "<"},
|
||||
"quot" : { regex: /&(quot|#34|#x22);/g, val : "\""},
|
||||
};
|
||||
this.ampEntity = { regex: /&(amp|#38|#x26);/g, val : "&"};
|
||||
this.htmlEntities = {
|
||||
"space": { regex: /&(nbsp|#160);/g, val: " " },
|
||||
// "lt" : { regex: /&(lt|#60);/g, val: "<" },
|
||||
// "gt" : { regex: /&(gt|#62);/g, val: ">" },
|
||||
// "amp" : { regex: /&(amp|#38);/g, val: "&" },
|
||||
// "quot" : { regex: /&(quot|#34);/g, val: "\"" },
|
||||
// "apos" : { regex: /&(apos|#39);/g, val: "'" },
|
||||
"cent" : { regex: /&(cent|#162);/g, val: "¢" },
|
||||
"pound" : { regex: /&(pound|#163);/g, val: "£" },
|
||||
"yen" : { regex: /&(yen|#165);/g, val: "¥" },
|
||||
"euro" : { regex: /&(euro|#8364);/g, val: "€" },
|
||||
"copyright" : { regex: /&(copy|#169);/g, val: "©" },
|
||||
"reg" : { regex: /&(reg|#174);/g, val: "®" },
|
||||
"inr" : { regex: /&(inr|#8377);/g, val: "₹" },
|
||||
};
|
||||
this.addExternalEntities = addExternalEntities;
|
||||
this.parseXml = parseXml;
|
||||
this.parseTextData = parseTextData;
|
||||
this.resolveNameSpace = resolveNameSpace;
|
||||
this.buildAttributesMap = buildAttributesMap;
|
||||
this.isItStopNode = isItStopNode;
|
||||
this.replaceEntitiesValue = replaceEntitiesValue;
|
||||
this.readStopNodeData = readStopNodeData;
|
||||
this.saveTextToParentTag = saveTextToParentTag;
|
||||
this.addChild = addChild;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function addExternalEntities(externalEntities){
|
||||
const entKeys = Object.keys(externalEntities);
|
||||
for (let i = 0; i < entKeys.length; i++) {
|
||||
const ent = entKeys[i];
|
||||
this.lastEntities[ent] = {
|
||||
regex: new RegExp("&"+ent+";","g"),
|
||||
val : externalEntities[ent]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} val
|
||||
* @param {string} tagName
|
||||
* @param {string} jPath
|
||||
* @param {boolean} dontTrim
|
||||
* @param {boolean} hasAttributes
|
||||
* @param {boolean} isLeafNode
|
||||
* @param {boolean} escapeEntities
|
||||
*/
|
||||
function parseTextData(val, tagName, jPath, dontTrim, hasAttributes, isLeafNode, escapeEntities) {
|
||||
if (val !== undefined) {
|
||||
if (this.options.trimValues && !dontTrim) {
|
||||
val = val.trim();
|
||||
}
|
||||
if(val.length > 0){
|
||||
if(!escapeEntities) val = this.replaceEntitiesValue(val);
|
||||
|
||||
const newval = this.options.tagValueProcessor(tagName, val, jPath, hasAttributes, isLeafNode);
|
||||
if(newval === null || newval === undefined){
|
||||
//don't parse
|
||||
return val;
|
||||
}else if(typeof newval !== typeof val || newval !== val){
|
||||
//overwrite
|
||||
return newval;
|
||||
}else if(this.options.trimValues){
|
||||
return parseValue(val, this.options.parseTagValue, this.options.numberParseOptions);
|
||||
}else{
|
||||
const trimmedVal = val.trim();
|
||||
if(trimmedVal === val){
|
||||
return parseValue(val, this.options.parseTagValue, this.options.numberParseOptions);
|
||||
}else{
|
||||
return val;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resolveNameSpace(tagname) {
|
||||
if (this.options.removeNSPrefix) {
|
||||
const tags = tagname.split(':');
|
||||
const prefix = tagname.charAt(0) === '/' ? '/' : '';
|
||||
if (tags[0] === 'xmlns') {
|
||||
return '';
|
||||
}
|
||||
if (tags.length === 2) {
|
||||
tagname = prefix + tags[1];
|
||||
}
|
||||
}
|
||||
return tagname;
|
||||
}
|
||||
|
||||
//TODO: change regex to capture NS
|
||||
//const attrsRegx = new RegExp("([\\w\\-\\.\\:]+)\\s*=\\s*(['\"])((.|\n)*?)\\2","gm");
|
||||
const attrsRegx = new RegExp('([^\\s=]+)\\s*(=\\s*([\'"])([\\s\\S]*?)\\3)?', 'gm');
|
||||
|
||||
function buildAttributesMap(attrStr, jPath, tagName) {
|
||||
if (!this.options.ignoreAttributes && typeof attrStr === 'string') {
|
||||
// attrStr = attrStr.replace(/\r?\n/g, ' ');
|
||||
//attrStr = attrStr || attrStr.trim();
|
||||
|
||||
const matches = util.getAllMatches(attrStr, attrsRegx);
|
||||
const len = matches.length; //don't make it inline
|
||||
const attrs = {};
|
||||
for (let i = 0; i < len; i++) {
|
||||
const attrName = this.resolveNameSpace(matches[i][1]);
|
||||
let oldVal = matches[i][4];
|
||||
let aName = this.options.attributeNamePrefix + attrName;
|
||||
if (attrName.length) {
|
||||
if (this.options.transformAttributeName) {
|
||||
aName = this.options.transformAttributeName(aName);
|
||||
}
|
||||
if(aName === "__proto__") aName = "#__proto__";
|
||||
if (oldVal !== undefined) {
|
||||
if (this.options.trimValues) {
|
||||
oldVal = oldVal.trim();
|
||||
}
|
||||
oldVal = this.replaceEntitiesValue(oldVal);
|
||||
const newVal = this.options.attributeValueProcessor(attrName, oldVal, jPath);
|
||||
if(newVal === null || newVal === undefined){
|
||||
//don't parse
|
||||
attrs[aName] = oldVal;
|
||||
}else if(typeof newVal !== typeof oldVal || newVal !== oldVal){
|
||||
//overwrite
|
||||
attrs[aName] = newVal;
|
||||
}else{
|
||||
//parse
|
||||
attrs[aName] = parseValue(
|
||||
oldVal,
|
||||
this.options.parseAttributeValue,
|
||||
this.options.numberParseOptions
|
||||
);
|
||||
}
|
||||
} else if (this.options.allowBooleanAttributes) {
|
||||
attrs[aName] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!Object.keys(attrs).length) {
|
||||
return;
|
||||
}
|
||||
if (this.options.attributesGroupName) {
|
||||
const attrCollection = {};
|
||||
attrCollection[this.options.attributesGroupName] = attrs;
|
||||
return attrCollection;
|
||||
}
|
||||
return attrs
|
||||
}
|
||||
}
|
||||
|
||||
const parseXml = function(xmlData) {
|
||||
xmlData = xmlData.replace(/\r\n?/g, "\n"); //TODO: remove this line
|
||||
const xmlObj = new xmlNode('!xml');
|
||||
let currentNode = xmlObj;
|
||||
let textData = "";
|
||||
let jPath = "";
|
||||
for(let i=0; i< xmlData.length; i++){//for each char in XML data
|
||||
const ch = xmlData[i];
|
||||
if(ch === '<'){
|
||||
// const nextIndex = i+1;
|
||||
// const _2ndChar = xmlData[nextIndex];
|
||||
if( xmlData[i+1] === '/') {//Closing Tag
|
||||
const closeIndex = findClosingIndex(xmlData, ">", i, "Closing Tag is not closed.")
|
||||
let tagName = xmlData.substring(i+2,closeIndex).trim();
|
||||
|
||||
if(this.options.removeNSPrefix){
|
||||
const colonIndex = tagName.indexOf(":");
|
||||
if(colonIndex !== -1){
|
||||
tagName = tagName.substr(colonIndex+1);
|
||||
}
|
||||
}
|
||||
|
||||
if(this.options.transformTagName) {
|
||||
tagName = this.options.transformTagName(tagName);
|
||||
}
|
||||
|
||||
if(currentNode){
|
||||
textData = this.saveTextToParentTag(textData, currentNode, jPath);
|
||||
}
|
||||
|
||||
//check if last tag of nested tag was unpaired tag
|
||||
const lastTagName = jPath.substring(jPath.lastIndexOf(".")+1);
|
||||
if(tagName && this.options.unpairedTags.indexOf(tagName) !== -1 ){
|
||||
throw new Error(`Unpaired tag can not be used as closing tag: </${tagName}>`);
|
||||
}
|
||||
let propIndex = 0
|
||||
if(lastTagName && this.options.unpairedTags.indexOf(lastTagName) !== -1 ){
|
||||
propIndex = jPath.lastIndexOf('.', jPath.lastIndexOf('.')-1)
|
||||
this.tagsNodeStack.pop();
|
||||
}else{
|
||||
propIndex = jPath.lastIndexOf(".");
|
||||
}
|
||||
jPath = jPath.substring(0, propIndex);
|
||||
|
||||
currentNode = this.tagsNodeStack.pop();//avoid recursion, set the parent tag scope
|
||||
textData = "";
|
||||
i = closeIndex;
|
||||
} else if( xmlData[i+1] === '?') {
|
||||
|
||||
let tagData = readTagExp(xmlData,i, false, "?>");
|
||||
if(!tagData) throw new Error("Pi Tag is not closed.");
|
||||
|
||||
textData = this.saveTextToParentTag(textData, currentNode, jPath);
|
||||
if( (this.options.ignoreDeclaration && tagData.tagName === "?xml") || this.options.ignorePiTags){
|
||||
|
||||
}else{
|
||||
|
||||
const childNode = new xmlNode(tagData.tagName);
|
||||
childNode.add(this.options.textNodeName, "");
|
||||
|
||||
if(tagData.tagName !== tagData.tagExp && tagData.attrExpPresent){
|
||||
childNode[":@"] = this.buildAttributesMap(tagData.tagExp, jPath, tagData.tagName);
|
||||
}
|
||||
this.addChild(currentNode, childNode, jPath)
|
||||
|
||||
}
|
||||
|
||||
|
||||
i = tagData.closeIndex + 1;
|
||||
} else if(xmlData.substr(i + 1, 3) === '!--') {
|
||||
const endIndex = findClosingIndex(xmlData, "-->", i+4, "Comment is not closed.")
|
||||
if(this.options.commentPropName){
|
||||
const comment = xmlData.substring(i + 4, endIndex - 2);
|
||||
|
||||
textData = this.saveTextToParentTag(textData, currentNode, jPath);
|
||||
|
||||
currentNode.add(this.options.commentPropName, [ { [this.options.textNodeName] : comment } ]);
|
||||
}
|
||||
i = endIndex;
|
||||
} else if( xmlData.substr(i + 1, 2) === '!D') {
|
||||
const result = readDocType(xmlData, i);
|
||||
this.docTypeEntities = result.entities;
|
||||
i = result.i;
|
||||
}else if(xmlData.substr(i + 1, 2) === '![') {
|
||||
const closeIndex = findClosingIndex(xmlData, "]]>", i, "CDATA is not closed.") - 2;
|
||||
const tagExp = xmlData.substring(i + 9,closeIndex);
|
||||
|
||||
textData = this.saveTextToParentTag(textData, currentNode, jPath);
|
||||
|
||||
//cdata should be set even if it is 0 length string
|
||||
if(this.options.cdataPropName){
|
||||
// let val = this.parseTextData(tagExp, this.options.cdataPropName, jPath + "." + this.options.cdataPropName, true, false, true);
|
||||
// if(!val) val = "";
|
||||
currentNode.add(this.options.cdataPropName, [ { [this.options.textNodeName] : tagExp } ]);
|
||||
}else{
|
||||
let val = this.parseTextData(tagExp, currentNode.tagname, jPath, true, false, true);
|
||||
if(val == undefined) val = "";
|
||||
currentNode.add(this.options.textNodeName, val);
|
||||
}
|
||||
|
||||
i = closeIndex + 2;
|
||||
}else {//Opening tag
|
||||
let result = readTagExp(xmlData,i, this.options.removeNSPrefix);
|
||||
let tagName= result.tagName;
|
||||
const rawTagName = result.rawTagName;
|
||||
let tagExp = result.tagExp;
|
||||
let attrExpPresent = result.attrExpPresent;
|
||||
let closeIndex = result.closeIndex;
|
||||
|
||||
if (this.options.transformTagName) {
|
||||
tagName = this.options.transformTagName(tagName);
|
||||
}
|
||||
|
||||
//save text as child node
|
||||
if (currentNode && textData) {
|
||||
if(currentNode.tagname !== '!xml'){
|
||||
//when nested tag is found
|
||||
textData = this.saveTextToParentTag(textData, currentNode, jPath, false);
|
||||
}
|
||||
}
|
||||
|
||||
//check if last tag was unpaired tag
|
||||
const lastTag = currentNode;
|
||||
if(lastTag && this.options.unpairedTags.indexOf(lastTag.tagname) !== -1 ){
|
||||
currentNode = this.tagsNodeStack.pop();
|
||||
jPath = jPath.substring(0, jPath.lastIndexOf("."));
|
||||
}
|
||||
if(tagName !== xmlObj.tagname){
|
||||
jPath += jPath ? "." + tagName : tagName;
|
||||
}
|
||||
if (this.isItStopNode(this.options.stopNodes, jPath, tagName)) {
|
||||
let tagContent = "";
|
||||
//self-closing tag
|
||||
if(tagExp.length > 0 && tagExp.lastIndexOf("/") === tagExp.length - 1){
|
||||
i = result.closeIndex;
|
||||
}
|
||||
//unpaired tag
|
||||
else if(this.options.unpairedTags.indexOf(tagName) !== -1){
|
||||
i = result.closeIndex;
|
||||
}
|
||||
//normal tag
|
||||
else{
|
||||
//read until closing tag is found
|
||||
const result = this.readStopNodeData(xmlData, rawTagName, closeIndex + 1);
|
||||
if(!result) throw new Error(`Unexpected end of ${rawTagName}`);
|
||||
i = result.i;
|
||||
tagContent = result.tagContent;
|
||||
}
|
||||
|
||||
const childNode = new xmlNode(tagName);
|
||||
if(tagName !== tagExp && attrExpPresent){
|
||||
childNode[":@"] = this.buildAttributesMap(tagExp, jPath, tagName);
|
||||
}
|
||||
if(tagContent) {
|
||||
tagContent = this.parseTextData(tagContent, tagName, jPath, true, attrExpPresent, true, true);
|
||||
}
|
||||
|
||||
jPath = jPath.substr(0, jPath.lastIndexOf("."));
|
||||
childNode.add(this.options.textNodeName, tagContent);
|
||||
|
||||
this.addChild(currentNode, childNode, jPath)
|
||||
}else{
|
||||
//selfClosing tag
|
||||
if(tagExp.length > 0 && tagExp.lastIndexOf("/") === tagExp.length - 1){
|
||||
if(tagName[tagName.length - 1] === "/"){ //remove trailing '/'
|
||||
tagName = tagName.substr(0, tagName.length - 1);
|
||||
jPath = jPath.substr(0, jPath.length - 1);
|
||||
tagExp = tagName;
|
||||
}else{
|
||||
tagExp = tagExp.substr(0, tagExp.length - 1);
|
||||
}
|
||||
|
||||
if(this.options.transformTagName) {
|
||||
tagName = this.options.transformTagName(tagName);
|
||||
}
|
||||
|
||||
const childNode = new xmlNode(tagName);
|
||||
if(tagName !== tagExp && attrExpPresent){
|
||||
childNode[":@"] = this.buildAttributesMap(tagExp, jPath, tagName);
|
||||
}
|
||||
this.addChild(currentNode, childNode, jPath)
|
||||
jPath = jPath.substr(0, jPath.lastIndexOf("."));
|
||||
}
|
||||
//opening tag
|
||||
else{
|
||||
const childNode = new xmlNode( tagName);
|
||||
this.tagsNodeStack.push(currentNode);
|
||||
|
||||
if(tagName !== tagExp && attrExpPresent){
|
||||
childNode[":@"] = this.buildAttributesMap(tagExp, jPath, tagName);
|
||||
}
|
||||
this.addChild(currentNode, childNode, jPath)
|
||||
currentNode = childNode;
|
||||
}
|
||||
textData = "";
|
||||
i = closeIndex;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
textData += xmlData[i];
|
||||
}
|
||||
}
|
||||
return xmlObj.child;
|
||||
}
|
||||
|
||||
function addChild(currentNode, childNode, jPath){
|
||||
const result = this.options.updateTag(childNode.tagname, jPath, childNode[":@"])
|
||||
if(result === false){
|
||||
}else if(typeof result === "string"){
|
||||
childNode.tagname = result
|
||||
currentNode.addChild(childNode);
|
||||
}else{
|
||||
currentNode.addChild(childNode);
|
||||
}
|
||||
}
|
||||
|
||||
const replaceEntitiesValue = function(val){
|
||||
|
||||
if(this.options.processEntities){
|
||||
for(let entityName in this.docTypeEntities){
|
||||
const entity = this.docTypeEntities[entityName];
|
||||
val = val.replace( entity.regx, entity.val);
|
||||
}
|
||||
for(let entityName in this.lastEntities){
|
||||
const entity = this.lastEntities[entityName];
|
||||
val = val.replace( entity.regex, entity.val);
|
||||
}
|
||||
if(this.options.htmlEntities){
|
||||
for(let entityName in this.htmlEntities){
|
||||
const entity = this.htmlEntities[entityName];
|
||||
val = val.replace( entity.regex, entity.val);
|
||||
}
|
||||
}
|
||||
val = val.replace( this.ampEntity.regex, this.ampEntity.val);
|
||||
}
|
||||
return val;
|
||||
}
|
||||
function saveTextToParentTag(textData, currentNode, jPath, isLeafNode) {
|
||||
if (textData) { //store previously collected data as textNode
|
||||
if(isLeafNode === undefined) isLeafNode = Object.keys(currentNode.child).length === 0
|
||||
|
||||
textData = this.parseTextData(textData,
|
||||
currentNode.tagname,
|
||||
jPath,
|
||||
false,
|
||||
currentNode[":@"] ? Object.keys(currentNode[":@"]).length !== 0 : false,
|
||||
isLeafNode);
|
||||
|
||||
if (textData !== undefined && textData !== "")
|
||||
currentNode.add(this.options.textNodeName, textData);
|
||||
textData = "";
|
||||
}
|
||||
return textData;
|
||||
}
|
||||
|
||||
//TODO: use jPath to simplify the logic
|
||||
/**
|
||||
*
|
||||
* @param {string[]} stopNodes
|
||||
* @param {string} jPath
|
||||
* @param {string} currentTagName
|
||||
*/
|
||||
function isItStopNode(stopNodes, jPath, currentTagName){
|
||||
const allNodesExp = "*." + currentTagName;
|
||||
for (const stopNodePath in stopNodes) {
|
||||
const stopNodeExp = stopNodes[stopNodePath];
|
||||
if( allNodesExp === stopNodeExp || jPath === stopNodeExp ) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the tag Expression and where it is ending handling single-double quotes situation
|
||||
* @param {string} xmlData
|
||||
* @param {number} i starting index
|
||||
* @returns
|
||||
*/
|
||||
function tagExpWithClosingIndex(xmlData, i, closingChar = ">"){
|
||||
let attrBoundary;
|
||||
let tagExp = "";
|
||||
for (let index = i; index < xmlData.length; index++) {
|
||||
let ch = xmlData[index];
|
||||
if (attrBoundary) {
|
||||
if (ch === attrBoundary) attrBoundary = "";//reset
|
||||
} else if (ch === '"' || ch === "'") {
|
||||
attrBoundary = ch;
|
||||
} else if (ch === closingChar[0]) {
|
||||
if(closingChar[1]){
|
||||
if(xmlData[index + 1] === closingChar[1]){
|
||||
return {
|
||||
data: tagExp,
|
||||
index: index
|
||||
}
|
||||
}
|
||||
}else{
|
||||
return {
|
||||
data: tagExp,
|
||||
index: index
|
||||
}
|
||||
}
|
||||
} else if (ch === '\t') {
|
||||
ch = " "
|
||||
}
|
||||
tagExp += ch;
|
||||
}
|
||||
}
|
||||
|
||||
function findClosingIndex(xmlData, str, i, errMsg){
|
||||
const closingIndex = xmlData.indexOf(str, i);
|
||||
if(closingIndex === -1){
|
||||
throw new Error(errMsg)
|
||||
}else{
|
||||
return closingIndex + str.length - 1;
|
||||
}
|
||||
}
|
||||
|
||||
function readTagExp(xmlData,i, removeNSPrefix, closingChar = ">"){
|
||||
const result = tagExpWithClosingIndex(xmlData, i+1, closingChar);
|
||||
if(!result) return;
|
||||
let tagExp = result.data;
|
||||
const closeIndex = result.index;
|
||||
const separatorIndex = tagExp.search(/\s/);
|
||||
let tagName = tagExp;
|
||||
let attrExpPresent = true;
|
||||
if(separatorIndex !== -1){//separate tag name and attributes expression
|
||||
tagName = tagExp.substr(0, separatorIndex).replace(/\s\s*$/, '');
|
||||
tagExp = tagExp.substr(separatorIndex + 1);
|
||||
}
|
||||
|
||||
const rawTagName = tagName;
|
||||
if(removeNSPrefix){
|
||||
const colonIndex = tagName.indexOf(":");
|
||||
if(colonIndex !== -1){
|
||||
tagName = tagName.substr(colonIndex+1);
|
||||
attrExpPresent = tagName !== result.data.substr(colonIndex + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
tagName: tagName,
|
||||
tagExp: tagExp,
|
||||
closeIndex: closeIndex,
|
||||
attrExpPresent: attrExpPresent,
|
||||
rawTagName: rawTagName,
|
||||
}
|
||||
}
|
||||
/**
|
||||
* find paired tag for a stop node
|
||||
* @param {string} xmlData
|
||||
* @param {string} tagName
|
||||
* @param {number} i
|
||||
*/
|
||||
function readStopNodeData(xmlData, tagName, i){
|
||||
const startIndex = i;
|
||||
// Starting at 1 since we already have an open tag
|
||||
let openTagCount = 1;
|
||||
|
||||
for (; i < xmlData.length; i++) {
|
||||
if( xmlData[i] === "<"){
|
||||
if (xmlData[i+1] === "/") {//close tag
|
||||
const closeIndex = findClosingIndex(xmlData, ">", i, `${tagName} is not closed`);
|
||||
let closeTagName = xmlData.substring(i+2,closeIndex).trim();
|
||||
if(closeTagName === tagName){
|
||||
openTagCount--;
|
||||
if (openTagCount === 0) {
|
||||
return {
|
||||
tagContent: xmlData.substring(startIndex, i),
|
||||
i : closeIndex
|
||||
}
|
||||
}
|
||||
}
|
||||
i=closeIndex;
|
||||
} else if(xmlData[i+1] === '?') {
|
||||
const closeIndex = findClosingIndex(xmlData, "?>", i+1, "StopNode is not closed.")
|
||||
i=closeIndex;
|
||||
} else if(xmlData.substr(i + 1, 3) === '!--') {
|
||||
const closeIndex = findClosingIndex(xmlData, "-->", i+3, "StopNode is not closed.")
|
||||
i=closeIndex;
|
||||
} else if(xmlData.substr(i + 1, 2) === '![') {
|
||||
const closeIndex = findClosingIndex(xmlData, "]]>", i, "StopNode is not closed.") - 2;
|
||||
i=closeIndex;
|
||||
} else {
|
||||
const tagData = readTagExp(xmlData, i, '>')
|
||||
|
||||
if (tagData) {
|
||||
const openTagName = tagData && tagData.tagName;
|
||||
if (openTagName === tagName && tagData.tagExp[tagData.tagExp.length-1] !== "/") {
|
||||
openTagCount++;
|
||||
}
|
||||
i=tagData.closeIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
}//end for loop
|
||||
}
|
||||
|
||||
function parseValue(val, shouldParse, options) {
|
||||
if (shouldParse && typeof val === 'string') {
|
||||
//console.log(options)
|
||||
const newval = val.trim();
|
||||
if(newval === 'true' ) return true;
|
||||
else if(newval === 'false' ) return false;
|
||||
else return toNumber(val, options);
|
||||
} else {
|
||||
if (util.isExist(val)) {
|
||||
return val;
|
||||
} else {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
module.exports = OrderedObjParser;
|
||||
Generated
Vendored
-58
@@ -1,58 +0,0 @@
|
||||
const { buildOptions} = require("./OptionsBuilder");
|
||||
const OrderedObjParser = require("./OrderedObjParser");
|
||||
const { prettify} = require("./node2json");
|
||||
const validator = require('../validator');
|
||||
|
||||
class XMLParser{
|
||||
|
||||
constructor(options){
|
||||
this.externalEntities = {};
|
||||
this.options = buildOptions(options);
|
||||
|
||||
}
|
||||
/**
|
||||
* Parse XML dats to JS object
|
||||
* @param {string|Buffer} xmlData
|
||||
* @param {boolean|Object} validationOption
|
||||
*/
|
||||
parse(xmlData,validationOption){
|
||||
if(typeof xmlData === "string"){
|
||||
}else if( xmlData.toString){
|
||||
xmlData = xmlData.toString();
|
||||
}else{
|
||||
throw new Error("XML data is accepted in String or Bytes[] form.")
|
||||
}
|
||||
if( validationOption){
|
||||
if(validationOption === true) validationOption = {}; //validate with default options
|
||||
|
||||
const result = validator.validate(xmlData, validationOption);
|
||||
if (result !== true) {
|
||||
throw Error( `${result.err.msg}:${result.err.line}:${result.err.col}` )
|
||||
}
|
||||
}
|
||||
const orderedObjParser = new OrderedObjParser(this.options);
|
||||
orderedObjParser.addExternalEntities(this.externalEntities);
|
||||
const orderedResult = orderedObjParser.parseXml(xmlData);
|
||||
if(this.options.preserveOrder || orderedResult === undefined) return orderedResult;
|
||||
else return prettify(orderedResult, this.options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add Entity which is not by default supported by this library
|
||||
* @param {string} key
|
||||
* @param {string} value
|
||||
*/
|
||||
addEntity(key, value){
|
||||
if(value.indexOf("&") !== -1){
|
||||
throw new Error("Entity value can't have '&'")
|
||||
}else if(key.indexOf("&") !== -1 || key.indexOf(";") !== -1){
|
||||
throw new Error("An entity must be set without '&' and ';'. Eg. use '#xD' for '
'")
|
||||
}else if(value === "&"){
|
||||
throw new Error("An entity with value '&' is not permitted");
|
||||
}else{
|
||||
this.externalEntities[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = XMLParser;
|
||||
Generated
Vendored
-113
@@ -1,113 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {array} node
|
||||
* @param {any} options
|
||||
* @returns
|
||||
*/
|
||||
function prettify(node, options){
|
||||
return compress( node, options);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {array} arr
|
||||
* @param {object} options
|
||||
* @param {string} jPath
|
||||
* @returns object
|
||||
*/
|
||||
function compress(arr, options, jPath){
|
||||
let text;
|
||||
const compressedObj = {};
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
const tagObj = arr[i];
|
||||
const property = propName(tagObj);
|
||||
let newJpath = "";
|
||||
if(jPath === undefined) newJpath = property;
|
||||
else newJpath = jPath + "." + property;
|
||||
|
||||
if(property === options.textNodeName){
|
||||
if(text === undefined) text = tagObj[property];
|
||||
else text += "" + tagObj[property];
|
||||
}else if(property === undefined){
|
||||
continue;
|
||||
}else if(tagObj[property]){
|
||||
|
||||
let val = compress(tagObj[property], options, newJpath);
|
||||
const isLeaf = isLeafTag(val, options);
|
||||
|
||||
if(tagObj[":@"]){
|
||||
assignAttributes( val, tagObj[":@"], newJpath, options);
|
||||
}else if(Object.keys(val).length === 1 && val[options.textNodeName] !== undefined && !options.alwaysCreateTextNode){
|
||||
val = val[options.textNodeName];
|
||||
}else if(Object.keys(val).length === 0){
|
||||
if(options.alwaysCreateTextNode) val[options.textNodeName] = "";
|
||||
else val = "";
|
||||
}
|
||||
|
||||
if(compressedObj[property] !== undefined && compressedObj.hasOwnProperty(property)) {
|
||||
if(!Array.isArray(compressedObj[property])) {
|
||||
compressedObj[property] = [ compressedObj[property] ];
|
||||
}
|
||||
compressedObj[property].push(val);
|
||||
}else{
|
||||
//TODO: if a node is not an array, then check if it should be an array
|
||||
//also determine if it is a leaf node
|
||||
if (options.isArray(property, newJpath, isLeaf )) {
|
||||
compressedObj[property] = [val];
|
||||
}else{
|
||||
compressedObj[property] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
// if(text && text.length > 0) compressedObj[options.textNodeName] = text;
|
||||
if(typeof text === "string"){
|
||||
if(text.length > 0) compressedObj[options.textNodeName] = text;
|
||||
}else if(text !== undefined) compressedObj[options.textNodeName] = text;
|
||||
return compressedObj;
|
||||
}
|
||||
|
||||
function propName(obj){
|
||||
const keys = Object.keys(obj);
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
const key = keys[i];
|
||||
if(key !== ":@") return key;
|
||||
}
|
||||
}
|
||||
|
||||
function assignAttributes(obj, attrMap, jpath, options){
|
||||
if (attrMap) {
|
||||
const keys = Object.keys(attrMap);
|
||||
const len = keys.length; //don't make it inline
|
||||
for (let i = 0; i < len; i++) {
|
||||
const atrrName = keys[i];
|
||||
if (options.isArray(atrrName, jpath + "." + atrrName, true, true)) {
|
||||
obj[atrrName] = [ attrMap[atrrName] ];
|
||||
} else {
|
||||
obj[atrrName] = attrMap[atrrName];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isLeafTag(obj, options){
|
||||
const { textNodeName } = options;
|
||||
const propCount = Object.keys(obj).length;
|
||||
|
||||
if (propCount === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
propCount === 1 &&
|
||||
(obj[textNodeName] || typeof obj[textNodeName] === "boolean" || obj[textNodeName] === 0)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
exports.prettify = prettify;
|
||||
Generated
Vendored
-25
@@ -1,25 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
class XmlNode{
|
||||
constructor(tagname) {
|
||||
this.tagname = tagname;
|
||||
this.child = []; //nested tags, text, cdata, comments in order
|
||||
this[":@"] = {}; //attributes map
|
||||
}
|
||||
add(key,val){
|
||||
// this.child.push( {name : key, val: val, isCdata: isCdata });
|
||||
if(key === "__proto__") key = "#__proto__";
|
||||
this.child.push( {[key]: val });
|
||||
}
|
||||
addChild(node) {
|
||||
if(node.tagname === "__proto__") node.tagname = "#__proto__";
|
||||
if(node[":@"] && Object.keys(node[":@"]).length > 0){
|
||||
this.child.push( { [node.tagname]: node.child, [":@"]: node[":@"] });
|
||||
}else{
|
||||
this.child.push( { [node.tagname]: node.child });
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
module.exports = XmlNode;
|
||||
-52
@@ -1,52 +0,0 @@
|
||||
{
|
||||
"version": "7.0.7",
|
||||
"name": "@extractus/feed-extractor",
|
||||
"description": "To read and normalize RSS/ATOM/JSON feed data",
|
||||
"homepage": "https://extractor-demos.pages.dev",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git@github.com:extractus/feed-extractor.git"
|
||||
},
|
||||
"author": "@extractus",
|
||||
"main": "./src/main.js",
|
||||
"type": "module",
|
||||
"imports": {
|
||||
"cross-fetch": "./src/deno/cross-fetch.js"
|
||||
},
|
||||
"browser": {
|
||||
"cross-fetch": "./src/deno/cross-fetch.js"
|
||||
},
|
||||
"types": "./index.d.ts",
|
||||
"engines": {
|
||||
"node": ">= 16"
|
||||
},
|
||||
"scripts": {
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "eslint --fix .",
|
||||
"pretest": "npm run lint",
|
||||
"test": "NODE_ENV=test NODE_OPTIONS=--experimental-vm-modules jest --verbose --coverage=true",
|
||||
"eval": "node eval",
|
||||
"reset": "node reset"
|
||||
},
|
||||
"dependencies": {
|
||||
"bellajs": "^11.1.2",
|
||||
"cross-fetch": "^4.0.0",
|
||||
"fast-xml-parser": "^4.3.2",
|
||||
"html-entities": "^2.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"eslint": "^8.53.0",
|
||||
"https-proxy-agent": "^7.0.2",
|
||||
"jest": "^29.7.0",
|
||||
"nock": "^13.3.8"
|
||||
},
|
||||
"keywords": [
|
||||
"extractor",
|
||||
"parser",
|
||||
"feed",
|
||||
"rss",
|
||||
"atom",
|
||||
"util"
|
||||
],
|
||||
"license": "MIT"
|
||||
}
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
// reset.js
|
||||
|
||||
import {
|
||||
existsSync,
|
||||
unlinkSync
|
||||
} from 'fs'
|
||||
|
||||
import { execSync } from 'child_process'
|
||||
|
||||
const dirs = [
|
||||
'dist',
|
||||
'docs',
|
||||
'.nyc_output',
|
||||
'coverage',
|
||||
'node_modules',
|
||||
'.nuxt',
|
||||
]
|
||||
|
||||
const files = [
|
||||
'yarn.lock',
|
||||
'pnpm-lock.yaml',
|
||||
'package-lock.json',
|
||||
'coverage.lcov',
|
||||
]
|
||||
|
||||
dirs.forEach((d) => {
|
||||
execSync(`rm -rf ${d}`)
|
||||
})
|
||||
|
||||
files.forEach((f) => {
|
||||
if (existsSync(f)) {
|
||||
unlinkSync(f)
|
||||
}
|
||||
})
|
||||
-2
@@ -1,2 +0,0 @@
|
||||
// cross-fetch.js
|
||||
export default fetch
|
||||
-74
@@ -1,74 +0,0 @@
|
||||
// main.js
|
||||
|
||||
import { isValid as isValidUrl } from './utils/linker.js'
|
||||
|
||||
import retrieve from './utils/retrieve.js'
|
||||
import { validate, xml2obj, isRSS, isAtom, isRdf } from './utils/xmlparser.js'
|
||||
import parseJsonFeed from './utils/parseJsonFeed.js'
|
||||
import parseRssFeed from './utils/parseRssFeed.js'
|
||||
import parseAtomFeed from './utils/parseAtomFeed.js'
|
||||
import parseRdfFeed from './utils/parseRdfFeed.js'
|
||||
|
||||
const getopt = (options = {}) => {
|
||||
const {
|
||||
normalization = true,
|
||||
descriptionMaxLen = 250,
|
||||
useISODateFormat = true,
|
||||
xmlParserOptions = {},
|
||||
baseUrl = '',
|
||||
getExtraFeedFields = () => ({}),
|
||||
getExtraEntryFields = () => ({}),
|
||||
} = options
|
||||
|
||||
return {
|
||||
normalization,
|
||||
descriptionMaxLen,
|
||||
useISODateFormat,
|
||||
xmlParserOptions,
|
||||
baseUrl,
|
||||
getExtraFeedFields,
|
||||
getExtraEntryFields,
|
||||
}
|
||||
}
|
||||
|
||||
export const extractFromJson = (json, options = {}) => {
|
||||
return parseJsonFeed(json, getopt(options))
|
||||
}
|
||||
|
||||
export const extractFromXml = (xml, options = {}) => {
|
||||
if (!validate(xml)) {
|
||||
throw new Error('The XML document is not well-formed')
|
||||
}
|
||||
|
||||
const opts = getopt(options)
|
||||
|
||||
const data = xml2obj(xml, opts.xmlParserOptions)
|
||||
|
||||
return isRSS(data)
|
||||
? parseRssFeed(data, opts)
|
||||
: isAtom(data)
|
||||
? parseAtomFeed(data, opts)
|
||||
: isRdf(data)
|
||||
? parseRdfFeed(data, opts)
|
||||
: null
|
||||
}
|
||||
|
||||
export const extract = async (url, options = {}, fetchOptions = {}) => {
|
||||
if (!isValidUrl(url)) {
|
||||
throw new Error('Input param must be a valid URL')
|
||||
}
|
||||
|
||||
const data = await retrieve(url, fetchOptions)
|
||||
if (!data.text && !data.json) {
|
||||
throw new Error(`Failed to load content from "${url}"`)
|
||||
}
|
||||
|
||||
const { type, json, text } = data
|
||||
|
||||
return type === 'json' ? extractFromJson(json, options) : extractFromXml(text, options)
|
||||
}
|
||||
|
||||
export const read = async (url, options, fetchOptions) => {
|
||||
console.warn('WARNING: read() is deprecated. Please use extract() instead!')
|
||||
return extract(url, options, fetchOptions)
|
||||
}
|
||||
-505
@@ -1,505 +0,0 @@
|
||||
// main.test
|
||||
/* eslint-env jest */
|
||||
|
||||
import { readFileSync } from 'fs'
|
||||
|
||||
import nock from 'nock'
|
||||
import { HttpsProxyAgent } from 'https-proxy-agent'
|
||||
|
||||
import { hasProperty, isString } from 'bellajs'
|
||||
|
||||
import { extract, extractFromXml, extractFromJson, read } from './main.js'
|
||||
import { isValid as isValidUrl } from './utils/linker.js'
|
||||
|
||||
const env = process.env || {}
|
||||
const PROXY_SERVER = env.PROXY_SERVER || ''
|
||||
|
||||
const feedAttrs = 'title link description generator language published entries'.split(' ')
|
||||
const entryAttrs = 'title link description published id'.split(' ')
|
||||
|
||||
const parseUrl = (url) => {
|
||||
const re = new URL(url)
|
||||
return {
|
||||
baseUrl: `${re.protocol}//${re.host}`,
|
||||
path: re.pathname,
|
||||
}
|
||||
}
|
||||
|
||||
const isValidDate = (d) => {
|
||||
return (new Date(d)).toString() !== 'Invalid Date'
|
||||
}
|
||||
|
||||
const validateProps = (entry) => {
|
||||
const { id, link, title, published, description } = entry
|
||||
return isString(description) &&
|
||||
isString(id) && id !== '' &&
|
||||
isString(title) && title !== '' &&
|
||||
isString(link) && isValidUrl(link) &&
|
||||
isString(published) && isValidDate(published)
|
||||
}
|
||||
|
||||
describe('test extract() function with common issues', () => {
|
||||
test('extract feed from a non-string link', () => {
|
||||
expect(extract([])).rejects.toThrow(new Error('Input param must be a valid URL'))
|
||||
})
|
||||
|
||||
test('extract feed from a 404 link', () => {
|
||||
const url = 'https://somewhere.xyz/alpha/beta'
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(404)
|
||||
expect(extract(url)).rejects.toThrow(new Error('Request failed with error code 404'))
|
||||
})
|
||||
|
||||
test('extract feed from empty xml', () => {
|
||||
const url = 'https://empty-source.elsewhere/rss'
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, '', {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
expect(extract(url)).rejects.toThrow(new Error(`Failed to load content from "${url}"`))
|
||||
})
|
||||
|
||||
test('extract feed from invalid xml', async () => {
|
||||
const url = 'https://averybad-source.elsewhere/rss'
|
||||
const xml = '<?xml version="1.0" encoding="UTF-8><noop><oops></ooops>'
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
expect(extract(url)).rejects.toThrow(new Error('The XML document is not well-formed'))
|
||||
})
|
||||
|
||||
test('extract feed from invalid json', async () => {
|
||||
const url = 'https://averybad-source.elsewhere/jsonfeed'
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, 'this is not json string', {
|
||||
'Content-Type': 'application/json',
|
||||
})
|
||||
expect(extract(url)).rejects.toThrow(new Error('Failed to convert data to JSON object'))
|
||||
})
|
||||
})
|
||||
|
||||
describe('test extract() standard feed', () => {
|
||||
test('extract rss feed from Google', async () => {
|
||||
const url = 'https://some-news-page.tld/rss'
|
||||
const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url)
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
})
|
||||
|
||||
test('extract atom feed from Google', async () => {
|
||||
const url = 'https://some-news-page.tld/atom'
|
||||
const xml = readFileSync('test-data/atom-feed-standard-realworld.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url)
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
})
|
||||
|
||||
test('extract atom feed from Google with extraFields', async () => {
|
||||
const url = 'https://some-news-page.tld/atom'
|
||||
const xml = readFileSync('test-data/atom-feed-standard-realworld.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
getExtraFeedFields: data => {
|
||||
return {
|
||||
author: data.author,
|
||||
}
|
||||
},
|
||||
getExtraEntryFields: data => {
|
||||
return {
|
||||
id: data.id,
|
||||
}
|
||||
},
|
||||
})
|
||||
expect(hasProperty(result, 'author')).toBe(true)
|
||||
expect(hasProperty(result.entries[0], 'id')).toBe(true)
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
})
|
||||
|
||||
test('extract rdf feed from Slashdot with extraFields', async () => {
|
||||
const url = 'https://some-news-page.tld/atom'
|
||||
const xml = readFileSync('test-data/rdf-standard.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
getExtraFeedFields: data => {
|
||||
return {
|
||||
subject: data['dc:subject'],
|
||||
}
|
||||
},
|
||||
getExtraEntryFields: data => {
|
||||
return {
|
||||
author: data['dc:creator'],
|
||||
}
|
||||
},
|
||||
})
|
||||
expect(hasProperty(result, 'subject')).toBe(true)
|
||||
expect(hasProperty(result.entries[0], 'author')).toBe(true)
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
})
|
||||
|
||||
test('extract atom feed which contains multi links', async () => {
|
||||
const url = 'https://some-news-page.tld/atom/multilinks'
|
||||
const xml = readFileSync('test-data/atom-multilinks.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url)
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
})
|
||||
|
||||
test('extract json feed from Micro.blog', async () => {
|
||||
const url = 'https://some-news-page.tld/json'
|
||||
const json = readFileSync('test-data/json-feed-standard-realworld.json', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, json, {
|
||||
'Content-Type': 'text/json',
|
||||
})
|
||||
const result = await extract(url)
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
})
|
||||
|
||||
test('extract json feed from Micro.blog with extra fields', async () => {
|
||||
const url = 'https://some-news-page.tld/json'
|
||||
const json = readFileSync('test-data/json-feed-standard-realworld.json', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, json, {
|
||||
'Content-Type': 'text/json',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
getExtraFeedFields: data => {
|
||||
return {
|
||||
icon: data.icon,
|
||||
}
|
||||
},
|
||||
getExtraEntryFields: data => {
|
||||
return {
|
||||
id: data.id,
|
||||
}
|
||||
},
|
||||
})
|
||||
expect(hasProperty(result, 'icon')).toBe(true)
|
||||
expect(hasProperty(result.entries[0], 'id')).toBe(true)
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
})
|
||||
|
||||
test('extract rss feed from huggingface.co (no link)', async () => {
|
||||
const url = 'https://huggingface.co/no-link/rss'
|
||||
const xml = readFileSync('test-data/rss-feed-miss-link.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url)
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
})
|
||||
|
||||
test('extract rss feed from medium.com (content:encoded)', async () => {
|
||||
const url = 'https://medium.com/feed/@ameliakusiak'
|
||||
const xml = readFileSync('test-data/medium-feed.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url)
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('test extract() with `useISODateFormat` option', () => {
|
||||
test('set `useISODateFormat` to false', async () => {
|
||||
const url = 'https://realworld-standard-feed.tld/rss'
|
||||
const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
useISODateFormat: false,
|
||||
})
|
||||
expect(result.published).toEqual('Thu, 28 Jul 2022 03:39:57 GMT')
|
||||
expect(result.entries[0].published).toEqual('Thu, 28 Jul 2022 02:43:00 GMT')
|
||||
})
|
||||
|
||||
test('set `useISODateFormat` to true', async () => {
|
||||
const url = 'https://realworld-standard-feed.tld/rss'
|
||||
const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
useISODateFormat: true,
|
||||
})
|
||||
expect(result.published).toEqual('2022-07-28T03:39:57.000Z')
|
||||
expect(result.entries[0].published).toEqual('2022-07-28T02:43:00.000Z')
|
||||
})
|
||||
})
|
||||
|
||||
describe('test extract() without normalization', () => {
|
||||
test('extract rss feed from Google', async () => {
|
||||
const url = 'https://some-news-page.tld/rss'
|
||||
const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
normalization: false,
|
||||
})
|
||||
expect(hasProperty(result, 'webMaster')).toBe(true)
|
||||
expect(hasProperty(result, 'item')).toBe(true)
|
||||
expect(hasProperty(result.item[0], 'source')).toBe(true)
|
||||
})
|
||||
test('extract rss feed from standard example', async () => {
|
||||
const url = 'https://some-news-page.tld/rss'
|
||||
const xml = readFileSync('test-data/rss-feed-standard.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
normalization: false,
|
||||
})
|
||||
expect(hasProperty(result, 'copyright')).toBe(true)
|
||||
expect(hasProperty(result, 'item')).toBe(true)
|
||||
expect(hasProperty(result.item, 'guid')).toBe(true)
|
||||
})
|
||||
|
||||
test('extract rdf feed from Slashdot without normalization', async () => {
|
||||
const url = 'https://some-news-page.tld/atom'
|
||||
const xml = readFileSync('test-data/rdf-standard.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
normalization: false,
|
||||
})
|
||||
expect(hasProperty(result.channel, 'syn:updateBase')).toBe(true)
|
||||
expect(hasProperty(result.channel, 'dc:rights')).toBe(true)
|
||||
expect(hasProperty(result, 'item')).toBe(true)
|
||||
expect(hasProperty(result.item[0], 'slash:department')).toBe(true)
|
||||
})
|
||||
|
||||
test('extract atom feed from Google', async () => {
|
||||
const url = 'https://some-news-page.tld/atom'
|
||||
const xml = readFileSync('test-data/atom-feed-standard-realworld.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
normalization: false,
|
||||
})
|
||||
expect(hasProperty(result, 'id')).toBe(true)
|
||||
expect(hasProperty(result, 'rights')).toBe(true)
|
||||
expect(hasProperty(result, 'entry')).toBe(true)
|
||||
expect(hasProperty(result.entry[0], 'updated')).toBe(true)
|
||||
})
|
||||
|
||||
test('extract atom feed from standard example', async () => {
|
||||
const url = 'https://some-news-page.tld/atom'
|
||||
const xml = readFileSync('test-data/atom-feed-standard.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
normalization: false,
|
||||
})
|
||||
expect(hasProperty(result, 'id')).toBe(true)
|
||||
expect(hasProperty(result, 'entry')).toBe(true)
|
||||
expect(hasProperty(result.entry, 'published')).toBe(true)
|
||||
expect(hasProperty(result.entry, 'updated')).toBe(true)
|
||||
expect(hasProperty(result.entry, 'summary')).toBe(true)
|
||||
expect(hasProperty(result.entry, 'content')).toBe(true)
|
||||
})
|
||||
|
||||
test('extract json feed from Micro.blog', async () => {
|
||||
const url = 'https://some-news-page.tld/json'
|
||||
const json = readFileSync('test-data/json-feed-standard-realworld.json', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, json, {
|
||||
'Content-Type': 'application/json',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
normalization: false,
|
||||
})
|
||||
expect(hasProperty(result, 'icon')).toBe(true)
|
||||
expect(hasProperty(result, 'favicon')).toBe(true)
|
||||
expect(hasProperty(result, 'items')).toBe(true)
|
||||
expect(hasProperty(result.items[0], 'tags')).toBe(true)
|
||||
expect(hasProperty(result.items[0], 'date_published')).toBe(true)
|
||||
})
|
||||
|
||||
test('extract rss podcast feed with enclosure tag', async () => {
|
||||
const url = 'https://some-podcast-page.tld/podcast/rss'
|
||||
const xml = readFileSync('test-data/podcast.rss', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, {
|
||||
normalization: false,
|
||||
})
|
||||
expect(hasProperty(result, 'itunes:owner')).toBe(true)
|
||||
expect(hasProperty(result.item[0], 'itunes:duration')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('test extract with `baseUrl` option', () => {
|
||||
test('extract rss feed from file', () => {
|
||||
const baseUrl = 'https://huggingface.co'
|
||||
const xml = readFileSync('test-data/rss-feed-miss-base-url.xml', 'utf8')
|
||||
const result = extractFromXml(xml, { baseUrl })
|
||||
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
expect(result.link).toBe(baseUrl + '/blog')
|
||||
expect(result.entries[0].link).toBe(baseUrl + '/blog/intro-graphml')
|
||||
})
|
||||
|
||||
test('extract rdf feed from file', () => {
|
||||
const baseUrl = 'https://slashdot.org'
|
||||
const xml = readFileSync('test-data/rdf-standard.xml', 'utf8')
|
||||
const result = extractFromXml(xml, { baseUrl })
|
||||
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
expect(result.link).toBe(baseUrl + '/')
|
||||
const firstItemLink = result.entries[0].link
|
||||
expect(firstItemLink.startsWith('https://tech.slashdot.org/story/23/08/23/2238246/spacex-')).toBe(true)
|
||||
})
|
||||
|
||||
test('extract json feed from file', () => {
|
||||
const baseUrl = 'https://www.jsonfeed.org'
|
||||
const json = readFileSync('test-data/json-feed-miss-base-url.json', 'utf8')
|
||||
const result = extractFromJson(JSON.parse(json), { baseUrl })
|
||||
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
|
||||
expect(result.link).toBe(baseUrl + '/')
|
||||
expect(result.entries[0].link).toBe(baseUrl + '/2020/08/07/json-feed-version.html')
|
||||
})
|
||||
|
||||
test('extract rss feed with url', async () => {
|
||||
const url = 'https://huggingface.co/blog/rss'
|
||||
const xml = readFileSync('test-data/rss-feed-miss-base-url.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await extract(url, { baseUrl })
|
||||
|
||||
feedAttrs.forEach((k) => {
|
||||
expect(hasProperty(result, k)).toBe(true)
|
||||
})
|
||||
|
||||
entryAttrs.forEach((k) => {
|
||||
expect(hasProperty(result.entries[0], k)).toBe(true)
|
||||
})
|
||||
|
||||
expect(validateProps(result.entries[0])).toBe(true)
|
||||
expect(result.link).toBe(baseUrl + '/blog')
|
||||
expect(result.entries[0].link).toBe(baseUrl + '/blog/intro-graphml')
|
||||
})
|
||||
})
|
||||
|
||||
if (PROXY_SERVER !== '') {
|
||||
describe('test extract live RSS via proxy server', () => {
|
||||
test('check if extract method works with proxy server', async () => {
|
||||
const url = 'https://news.google.com/rss'
|
||||
const result = await extract(url, {}, {
|
||||
agent: new HttpsProxyAgent(PROXY_SERVER),
|
||||
})
|
||||
expect(result.title).toContain('Google News')
|
||||
expect(result.entries.length).toBeGreaterThan(0)
|
||||
}, 10000)
|
||||
})
|
||||
}
|
||||
|
||||
describe('check old method read()', () => {
|
||||
test('ensure that depricated method read() still works', async () => {
|
||||
const url = 'https://realworld-standard-feed.tld/rss'
|
||||
const xml = readFileSync('test-data/rss-feed-standard-realworld.xml', 'utf8')
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, xml, {
|
||||
'Content-Type': 'application/xml',
|
||||
})
|
||||
const result = await read(url, {
|
||||
useISODateFormat: true,
|
||||
})
|
||||
expect(result.published).toEqual('2022-07-28T03:39:57.000Z')
|
||||
expect(result.entries[0].published).toEqual('2022-07-28T02:43:00.000Z')
|
||||
})
|
||||
})
|
||||
-93
@@ -1,93 +0,0 @@
|
||||
// utils -> linker
|
||||
|
||||
export const isValid = (url = '') => {
|
||||
try {
|
||||
const ourl = new URL(url)
|
||||
return ourl !== null && ourl.protocol.startsWith('http')
|
||||
} catch (err) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export const absolutify = (fullUrl = '', relativeUrl = '') => {
|
||||
try {
|
||||
const result = new URL(relativeUrl, fullUrl)
|
||||
return result.toString()
|
||||
} catch (err) {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
const blacklistKeys = [
|
||||
'CNDID',
|
||||
'__twitter_impression',
|
||||
'_hsenc',
|
||||
'_openstat',
|
||||
'action_object_map',
|
||||
'action_ref_map',
|
||||
'action_type_map',
|
||||
'amp',
|
||||
'fb_action_ids',
|
||||
'fb_action_types',
|
||||
'fb_ref',
|
||||
'fb_source',
|
||||
'fbclid',
|
||||
'ga_campaign',
|
||||
'ga_content',
|
||||
'ga_medium',
|
||||
'ga_place',
|
||||
'ga_source',
|
||||
'ga_term',
|
||||
'gs_l',
|
||||
'hmb_campaign',
|
||||
'hmb_medium',
|
||||
'hmb_source',
|
||||
'mbid',
|
||||
'mc_cid',
|
||||
'mc_eid',
|
||||
'mkt_tok',
|
||||
'referrer',
|
||||
'spJobID',
|
||||
'spMailingID',
|
||||
'spReportId',
|
||||
'spUserID',
|
||||
'utm_brand',
|
||||
'utm_campaign',
|
||||
'utm_cid',
|
||||
'utm_content',
|
||||
'utm_int',
|
||||
'utm_mailing',
|
||||
'utm_medium',
|
||||
'utm_name',
|
||||
'utm_place',
|
||||
'utm_pubreferrer',
|
||||
'utm_reader',
|
||||
'utm_social',
|
||||
'utm_source',
|
||||
'utm_swu',
|
||||
'utm_term',
|
||||
'utm_userid',
|
||||
'utm_viz_id',
|
||||
'wt_mc_o',
|
||||
'yclid',
|
||||
'WT.mc_id',
|
||||
'WT.mc_ev',
|
||||
'WT.srch',
|
||||
'pk_source',
|
||||
'pk_medium',
|
||||
'pk_campaign',
|
||||
]
|
||||
|
||||
export const purify = (url) => {
|
||||
try {
|
||||
const pureUrl = new URL(url)
|
||||
|
||||
blacklistKeys.forEach((key) => {
|
||||
pureUrl.searchParams.delete(key)
|
||||
})
|
||||
|
||||
return pureUrl.toString().replace(pureUrl.hash, '')
|
||||
} catch (err) {
|
||||
return null
|
||||
}
|
||||
}
|
||||
-138
@@ -1,138 +0,0 @@
|
||||
// linker.test
|
||||
/* eslint-env jest */
|
||||
|
||||
import { isValid, absolutify, purify } from './linker.js'
|
||||
|
||||
describe('test exported methods from `linker`', () => {
|
||||
const cases = [
|
||||
{
|
||||
url: 'https://www.23hq.com',
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
url: 'https://secure.actblue.com',
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
url: 'https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06',
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
url: 'http://192.168.1.199:8081/example/page',
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
url: 'ftp://192.168.1.199:8081/example/page',
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
url: '',
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
url: null,
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
url: { a: 'x' },
|
||||
expected: false,
|
||||
},
|
||||
]
|
||||
cases.forEach(({ url, expected }) => {
|
||||
test(`isValid("${url}") must return "${expected}"`, () => {
|
||||
const result = isValid(url)
|
||||
expect(result).toEqual(expected)
|
||||
})
|
||||
})
|
||||
|
||||
const entries = [
|
||||
{
|
||||
full: '',
|
||||
expected: '',
|
||||
},
|
||||
{
|
||||
relative: {},
|
||||
expected: '',
|
||||
},
|
||||
{
|
||||
full: 'https://some.where/article/abc-xyz',
|
||||
relative: 'category/page.html',
|
||||
expected: 'https://some.where/article/category/page.html',
|
||||
},
|
||||
{
|
||||
full: 'https://some.where/article/abc-xyz',
|
||||
relative: '../category/page.html',
|
||||
expected: 'https://some.where/category/page.html',
|
||||
},
|
||||
{
|
||||
full: 'https://some.where/blog/authors/article/abc-xyz',
|
||||
relative: '/category/page.html',
|
||||
expected: 'https://some.where/category/page.html',
|
||||
},
|
||||
{
|
||||
full: 'https://some.where/article/abc-xyz',
|
||||
expected: 'https://some.where/article/abc-xyz',
|
||||
},
|
||||
]
|
||||
entries.forEach((entry) => {
|
||||
const {
|
||||
full,
|
||||
relative,
|
||||
expected,
|
||||
} = entry
|
||||
test(`absolutify("${full}", "${relative}") must become "${expected}"`, () => {
|
||||
const result = absolutify(full, relative)
|
||||
expect(result).toEqual(expected)
|
||||
})
|
||||
})
|
||||
|
||||
test('test url purify() with invalid url', () => {
|
||||
const urls = [
|
||||
null,
|
||||
'',
|
||||
123,
|
||||
{},
|
||||
]
|
||||
urls.forEach((url) => {
|
||||
const result = purify(url)
|
||||
expect(result).toEqual(null)
|
||||
})
|
||||
})
|
||||
|
||||
test('test url purify() removing regular marketing params', () => {
|
||||
const entries = [
|
||||
{
|
||||
url: 'https://some.where/article/abc-xyz',
|
||||
expected: 'https://some.where/article/abc-xyz',
|
||||
},
|
||||
{
|
||||
url: 'https://some.where/article/abc-xyz#name,bob',
|
||||
expected: 'https://some.where/article/abc-xyz',
|
||||
},
|
||||
{
|
||||
url: 'https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer',
|
||||
expected: 'https://some.where/article/abc-xyz',
|
||||
},
|
||||
{
|
||||
url: 'https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer',
|
||||
expected: 'https://some.where/article/abc-xyz?q=3',
|
||||
},
|
||||
{
|
||||
url: 'https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer',
|
||||
expected: 'https://some.where/article/abc-xyz',
|
||||
},
|
||||
{
|
||||
url: 'https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer',
|
||||
expected: 'https://some.where/article/abc-xyz?q=3',
|
||||
},
|
||||
]
|
||||
entries.forEach((entry) => {
|
||||
const {
|
||||
url,
|
||||
expected,
|
||||
} = entry
|
||||
const result = purify(url)
|
||||
expect(result).toEqual(expected)
|
||||
})
|
||||
})
|
||||
})
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
// normalizer
|
||||
|
||||
import {
|
||||
isString,
|
||||
isObject,
|
||||
isArray,
|
||||
hasProperty,
|
||||
stripTags,
|
||||
truncate
|
||||
} from 'bellajs'
|
||||
|
||||
import { decode } from 'html-entities'
|
||||
|
||||
import { absolutify, isValid as isValidUrl, purify as purifyUrl } from './linker.js'
|
||||
|
||||
export const toISODateString = (dstr) => {
|
||||
try {
|
||||
return dstr ? (new Date(dstr)).toISOString() : ''
|
||||
} catch (err) {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
export const buildDescription = (val, maxlen = 0) => {
|
||||
const stripped = stripTags(String(val).trim().replace(/^<!\[CDATA\[|\]\]>$/g, ''))
|
||||
const text = maxlen > 0 ? truncate(stripped, maxlen) : stripped
|
||||
return text.replace(/\n+/g, ' ')
|
||||
}
|
||||
|
||||
export const getText = (val) => {
|
||||
const txt = isObject(val) ? (val._text || val['#text'] || val._cdata || val.$t) : val
|
||||
return txt ? decode(String(txt).trim()) : ''
|
||||
}
|
||||
|
||||
export const getLink = (val = [], id = '') => {
|
||||
if (isObject(id) && hasProperty(id, '@_isPermaLink') && id['@_isPermaLink'] === 'true') {
|
||||
return getText(id)
|
||||
}
|
||||
const getEntryLink = (links) => {
|
||||
const items = links.map((item) => {
|
||||
return getLink(item)
|
||||
})
|
||||
return items.length > 0 ? items[0] : ''
|
||||
}
|
||||
const url = isString(val)
|
||||
? getText(val)
|
||||
: isObject(val) && hasProperty(val, 'href')
|
||||
? getText(val.href)
|
||||
: isObject(val) && hasProperty(val, '@_href')
|
||||
? getText(val['@_href'])
|
||||
: isObject(val) && hasProperty(val, '@_url')
|
||||
? getText(val['@_url'])
|
||||
: isObject(val) && hasProperty(val, '_attributes')
|
||||
? getText(val._attributes.href)
|
||||
: isArray(val) ? getEntryLink(val) : ''
|
||||
|
||||
return url ? url : isValidUrl(id) ? id : ''
|
||||
}
|
||||
|
||||
export const getPureUrl = (url, id = '', baseUrl) => {
|
||||
const link = getLink(url, id)
|
||||
const pu = purifyUrl(link)
|
||||
|
||||
return link
|
||||
? pu
|
||||
? pu
|
||||
: absolutify(baseUrl, link)
|
||||
: ''
|
||||
}
|
||||
|
||||
const hash = (str) => Math.abs(str.split('').reduce((s, c) => Math.imul(31, s) + c.charCodeAt(0) | 0, 0)).toString(36)
|
||||
|
||||
export const getEntryId = (id, url, pubDate) => {
|
||||
return id ? getText(id) : hash(getPureUrl(url)) + '-' + (new Date(pubDate)).getTime()
|
||||
}
|
||||
|
||||
export const getEnclosure = (val) => {
|
||||
const url = hasProperty(val, '@_url') ? val['@_url'] : ''
|
||||
const type = hasProperty(val, '@_type') ? val['@_type'] : ''
|
||||
const length = Number(hasProperty(val, '@_length') ? val['@_length'] : 0)
|
||||
return !url || !type
|
||||
? null
|
||||
: {
|
||||
url,
|
||||
type,
|
||||
length,
|
||||
}
|
||||
}
|
||||
|
||||
const getCategory = (v) => {
|
||||
return isObject(v)
|
||||
? {
|
||||
text: getText(v),
|
||||
domain: v['@_domain'],
|
||||
}
|
||||
: v
|
||||
}
|
||||
|
||||
export const getOptionalTags = (val, key) => {
|
||||
if (key === 'source') {
|
||||
return {
|
||||
text: getText(val),
|
||||
url: getLink(val),
|
||||
}
|
||||
}
|
||||
if (key === 'category') {
|
||||
return isArray(val) ? val.map(getCategory) : getCategory(val)
|
||||
}
|
||||
if (key === 'enclosure') {
|
||||
return getEnclosure(val)
|
||||
}
|
||||
return val
|
||||
}
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
// normalizer.test
|
||||
/* eslint-env jest */
|
||||
|
||||
import { toISODateString } from './normalizer.js'
|
||||
|
||||
describe('test `normalizer` methods', () => {
|
||||
test('test toISODateString()', () => {
|
||||
expect(toISODateString('Thu, 28 Jul 2022 08:59:58 GMT')).toEqual('2022-07-28T08:59:58.000Z')
|
||||
expect(toISODateString('2022-07-28T02:43:00.000000000Z')).toEqual('2022-07-28T02:43:00.000Z')
|
||||
expect(toISODateString('')).toEqual('')
|
||||
expect(toISODateString('Thi, 280 Jul 2022 108:79:68 XMT')).toEqual('')
|
||||
})
|
||||
})
|
||||
-140
@@ -1,140 +0,0 @@
|
||||
// parseAtomFeed.js
|
||||
|
||||
// specs: https://datatracker.ietf.org/doc/html/rfc5023
|
||||
// refer: https://validator.w3.org/feed/docs/atom.html
|
||||
|
||||
import { isArray, hasProperty } from 'bellajs'
|
||||
|
||||
import {
|
||||
getText,
|
||||
toISODateString,
|
||||
buildDescription,
|
||||
getPureUrl,
|
||||
getEntryId
|
||||
} from './normalizer.js'
|
||||
|
||||
const transform = (item, options) => {
|
||||
const {
|
||||
useISODateFormat,
|
||||
descriptionMaxLen,
|
||||
baseUrl,
|
||||
getExtraEntryFields,
|
||||
} = options
|
||||
|
||||
const {
|
||||
id = '',
|
||||
title = '',
|
||||
issued = '',
|
||||
modified = '',
|
||||
updated = '',
|
||||
published = '',
|
||||
link = '',
|
||||
summary = '',
|
||||
content = '',
|
||||
} = item
|
||||
|
||||
const pubDate = updated || modified || published || issued
|
||||
const htmlContent = getText(summary || content)
|
||||
const entry = {
|
||||
id: getEntryId(id, link, pubDate),
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, id, baseUrl),
|
||||
published: useISODateFormat ? toISODateString(pubDate) : pubDate,
|
||||
description: buildDescription(htmlContent, descriptionMaxLen),
|
||||
}
|
||||
|
||||
const extraFields = getExtraEntryFields(item)
|
||||
|
||||
return {
|
||||
...entry,
|
||||
...extraFields,
|
||||
}
|
||||
}
|
||||
|
||||
const flatten = (feed, baseUrl) => {
|
||||
const {
|
||||
id,
|
||||
title = '',
|
||||
link = '',
|
||||
entry,
|
||||
} = feed
|
||||
|
||||
const entries = isArray(entry) ? entry : [entry]
|
||||
const items = entries.map((entry) => {
|
||||
const {
|
||||
id,
|
||||
title = '',
|
||||
link = '',
|
||||
summary = '',
|
||||
content = '',
|
||||
} = entry
|
||||
const item = {
|
||||
...entry,
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, id, baseUrl),
|
||||
}
|
||||
if (hasProperty(item, 'summary')) {
|
||||
item.summary = getText(summary)
|
||||
}
|
||||
if (hasProperty(item, 'content')) {
|
||||
item.content = getText(content)
|
||||
}
|
||||
return item
|
||||
})
|
||||
|
||||
const output = {
|
||||
...feed,
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, id, baseUrl),
|
||||
entry: isArray(entry) ? items : items[0],
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
const parseAtom = (data, options = {}) => {
|
||||
const {
|
||||
normalization,
|
||||
baseUrl,
|
||||
getExtraFeedFields,
|
||||
} = options
|
||||
|
||||
const feedData = data.feed
|
||||
|
||||
if (!normalization) {
|
||||
return flatten(feedData, baseUrl)
|
||||
}
|
||||
|
||||
const {
|
||||
id = '',
|
||||
title = '',
|
||||
link = '',
|
||||
subtitle = '',
|
||||
generator = '',
|
||||
language = '',
|
||||
updated = '',
|
||||
entry: item = [],
|
||||
} = feedData
|
||||
|
||||
const extraFields = getExtraFeedFields(feedData)
|
||||
|
||||
const items = isArray(item) ? item : [item]
|
||||
|
||||
const published = options.useISODateFormat ? toISODateString(updated) : updated
|
||||
|
||||
return {
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, id, baseUrl),
|
||||
description: subtitle,
|
||||
language,
|
||||
generator,
|
||||
published,
|
||||
...extraFields,
|
||||
entries: items.map((item) => {
|
||||
return transform(item, options)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export default (data, options = {}) => {
|
||||
return parseAtom(data, options)
|
||||
}
|
||||
-89
@@ -1,89 +0,0 @@
|
||||
// parseJsonFeed.js
|
||||
|
||||
// specs: https://www.jsonfeed.org/version/1.1/
|
||||
|
||||
import { isArray } from 'bellajs'
|
||||
|
||||
import {
|
||||
toISODateString,
|
||||
buildDescription,
|
||||
getEntryId
|
||||
} from './normalizer.js'
|
||||
|
||||
import { absolutify, purify as purifyUrl } from './linker.js'
|
||||
|
||||
const transform = (item, options) => {
|
||||
const {
|
||||
useISODateFormat,
|
||||
descriptionMaxLen,
|
||||
baseUrl,
|
||||
getExtraEntryFields,
|
||||
} = options
|
||||
|
||||
const {
|
||||
id = '',
|
||||
title = '',
|
||||
url: link = '',
|
||||
date_published: pubDate = '',
|
||||
summary = '',
|
||||
content_html: htmlContent = '',
|
||||
content_text: textContent = '',
|
||||
} = item
|
||||
|
||||
const published = useISODateFormat ? toISODateString(pubDate) : pubDate
|
||||
const extraFields = getExtraEntryFields(item)
|
||||
|
||||
const entry = {
|
||||
id: getEntryId(id, link, pubDate),
|
||||
title,
|
||||
link: purifyUrl(link) || absolutify(baseUrl, link),
|
||||
published,
|
||||
description: buildDescription(textContent || htmlContent || summary, descriptionMaxLen),
|
||||
}
|
||||
|
||||
return {
|
||||
...entry,
|
||||
...extraFields,
|
||||
}
|
||||
}
|
||||
|
||||
const parseJson = (data, options) => {
|
||||
const {
|
||||
normalization,
|
||||
baseUrl,
|
||||
getExtraFeedFields,
|
||||
} = options
|
||||
|
||||
if (!normalization) {
|
||||
return data
|
||||
}
|
||||
|
||||
const {
|
||||
title = '',
|
||||
home_page_url: homepageUrl = '',
|
||||
description = '',
|
||||
language = '',
|
||||
items: item = [],
|
||||
} = data
|
||||
|
||||
const extraFields = getExtraFeedFields(data)
|
||||
|
||||
const items = isArray(item) ? item : [item]
|
||||
|
||||
return {
|
||||
title,
|
||||
link: purifyUrl(homepageUrl) || absolutify(baseUrl, homepageUrl),
|
||||
description,
|
||||
language,
|
||||
published: '',
|
||||
generator: '',
|
||||
...extraFields,
|
||||
entries: items.map((item) => {
|
||||
return transform(item, options)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export default (data, options = {}) => {
|
||||
return parseJson(data, options)
|
||||
}
|
||||
-129
@@ -1,129 +0,0 @@
|
||||
// parseRssFeed.js
|
||||
|
||||
// specs: https://www.rssboard.org/rss-specification
|
||||
|
||||
import { isArray } from 'bellajs'
|
||||
|
||||
import {
|
||||
getText,
|
||||
toISODateString,
|
||||
buildDescription,
|
||||
getPureUrl,
|
||||
getEntryId
|
||||
} from './normalizer.js'
|
||||
|
||||
const transform = (item, options) => {
|
||||
const {
|
||||
useISODateFormat,
|
||||
descriptionMaxLen,
|
||||
baseUrl,
|
||||
getExtraEntryFields,
|
||||
} = options
|
||||
|
||||
const {
|
||||
guid = '',
|
||||
title = '',
|
||||
link = '',
|
||||
'dc:date': pubDate = '',
|
||||
description = '',
|
||||
'content:encoded': content = '',
|
||||
} = item
|
||||
|
||||
const published = useISODateFormat ? toISODateString(pubDate) : pubDate
|
||||
const htmlContent = getText(description || content)
|
||||
const entry = {
|
||||
id: getEntryId(guid, link, pubDate),
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, guid, baseUrl),
|
||||
published,
|
||||
description: buildDescription(htmlContent, descriptionMaxLen),
|
||||
}
|
||||
|
||||
const extraFields = getExtraEntryFields(item)
|
||||
|
||||
return {
|
||||
...entry,
|
||||
...extraFields,
|
||||
}
|
||||
}
|
||||
|
||||
const flatten = (feed, baseUrl) => {
|
||||
const {
|
||||
title = '',
|
||||
link = '',
|
||||
item,
|
||||
} = feed
|
||||
|
||||
const items = isArray(item) ? item : [item]
|
||||
const entries = items.map((entry) => {
|
||||
const {
|
||||
id,
|
||||
title = '',
|
||||
link = '',
|
||||
} = entry
|
||||
|
||||
const item = {
|
||||
...entry,
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, id, baseUrl),
|
||||
}
|
||||
|
||||
return item
|
||||
})
|
||||
|
||||
const output = {
|
||||
...feed,
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, baseUrl),
|
||||
item: isArray(item) ? entries : entries[0],
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
const parseRdf = (data, options = {}) => {
|
||||
const {
|
||||
normalization,
|
||||
baseUrl,
|
||||
getExtraFeedFields,
|
||||
} = options
|
||||
|
||||
const feedData = data['rdf:RDF']
|
||||
|
||||
if (!normalization) {
|
||||
return flatten(feedData, baseUrl)
|
||||
}
|
||||
|
||||
const {
|
||||
title = '',
|
||||
link = '',
|
||||
description = '',
|
||||
generator = '',
|
||||
'dc:language': language = '',
|
||||
'dc:date': lastBuildDate = '',
|
||||
} = feedData.channel
|
||||
|
||||
const { item } = feedData
|
||||
|
||||
const extraFields = getExtraFeedFields(feedData)
|
||||
|
||||
const items = isArray(item) ? item : [item]
|
||||
|
||||
const published = options.useISODateFormat ? toISODateString(lastBuildDate) : lastBuildDate
|
||||
|
||||
return {
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, '', baseUrl),
|
||||
description,
|
||||
language,
|
||||
generator,
|
||||
published,
|
||||
...extraFields,
|
||||
entries: items.map((item) => {
|
||||
return transform(item, options)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export default (data, options = {}) => {
|
||||
return parseRdf(data, options)
|
||||
}
|
||||
-144
@@ -1,144 +0,0 @@
|
||||
// parseRssFeed.js
|
||||
|
||||
// specs: https://www.rssboard.org/rss-specification
|
||||
|
||||
import { isArray, hasProperty } from 'bellajs'
|
||||
|
||||
import {
|
||||
getText,
|
||||
toISODateString,
|
||||
buildDescription,
|
||||
getPureUrl,
|
||||
getOptionalTags,
|
||||
getEntryId
|
||||
} from './normalizer.js'
|
||||
|
||||
const transform = (item, options) => {
|
||||
const {
|
||||
useISODateFormat,
|
||||
descriptionMaxLen,
|
||||
baseUrl,
|
||||
getExtraEntryFields,
|
||||
} = options
|
||||
|
||||
const {
|
||||
guid = '',
|
||||
title = '',
|
||||
link = '',
|
||||
pubDate = '',
|
||||
description = '',
|
||||
'content:encoded': content = '',
|
||||
} = item
|
||||
|
||||
const published = useISODateFormat ? toISODateString(pubDate) : pubDate
|
||||
const htmlContent = getText(description || content)
|
||||
const entry = {
|
||||
id: getEntryId(guid, link, pubDate),
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, guid, baseUrl),
|
||||
published,
|
||||
description: buildDescription(htmlContent, descriptionMaxLen),
|
||||
}
|
||||
|
||||
const extraFields = getExtraEntryFields(item)
|
||||
|
||||
return {
|
||||
...entry,
|
||||
...extraFields,
|
||||
}
|
||||
}
|
||||
|
||||
const flatten = (feed, baseUrl) => {
|
||||
const {
|
||||
title = '',
|
||||
link = '',
|
||||
item,
|
||||
} = feed
|
||||
|
||||
const items = isArray(item) ? item : [item]
|
||||
const entries = items.map((entry) => {
|
||||
const {
|
||||
id,
|
||||
title = '',
|
||||
link = '',
|
||||
} = entry
|
||||
|
||||
const item = {
|
||||
...entry,
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, id, baseUrl),
|
||||
}
|
||||
|
||||
const txtTags = 'guid description source'.split(' ')
|
||||
|
||||
txtTags.forEach((key) => {
|
||||
if (hasProperty(entry, key)) {
|
||||
item[key] = getText(entry[key])
|
||||
}
|
||||
})
|
||||
|
||||
const optionalProps = 'source category enclosure author image'.split(' ')
|
||||
optionalProps.forEach((key) => {
|
||||
if (hasProperty(item, key)) {
|
||||
entry[key] = getOptionalTags(item[key], key)
|
||||
}
|
||||
})
|
||||
|
||||
return item
|
||||
})
|
||||
|
||||
const output = {
|
||||
...feed,
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, baseUrl),
|
||||
item: isArray(item) ? entries : entries[0],
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
const parseRss = (data, options = {}) => {
|
||||
const {
|
||||
normalization,
|
||||
baseUrl,
|
||||
getExtraFeedFields,
|
||||
} = options
|
||||
|
||||
const feedData = data.rss.channel
|
||||
|
||||
if (!normalization) {
|
||||
return flatten(feedData, baseUrl)
|
||||
}
|
||||
|
||||
const {
|
||||
title = '',
|
||||
link = '',
|
||||
description = '',
|
||||
generator = '',
|
||||
language = '',
|
||||
lastBuildDate = '',
|
||||
item = [],
|
||||
} = feedData
|
||||
|
||||
const extraFields = getExtraFeedFields(feedData)
|
||||
|
||||
const items = isArray(item) ? item : [item]
|
||||
|
||||
const published = options.useISODateFormat ? toISODateString(lastBuildDate) : lastBuildDate
|
||||
|
||||
return {
|
||||
title: getText(title),
|
||||
link: getPureUrl(link, '', baseUrl),
|
||||
description,
|
||||
language,
|
||||
generator,
|
||||
published,
|
||||
...extraFields,
|
||||
entries: items.map((item) => {
|
||||
return transform(item, options)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export default (data, options = {}) => {
|
||||
return parseRss(data, options)
|
||||
}
|
||||
-50
@@ -1,50 +0,0 @@
|
||||
// utils -> retrieve
|
||||
|
||||
import fetch from 'cross-fetch'
|
||||
|
||||
const profetch = async (url, options = {}) => {
|
||||
const { proxy = {}, signal = null } = options
|
||||
const {
|
||||
target,
|
||||
headers = {},
|
||||
} = proxy
|
||||
const res = await fetch(target + encodeURIComponent(url), {
|
||||
headers,
|
||||
signal,
|
||||
})
|
||||
return res
|
||||
}
|
||||
|
||||
export default async (url, options = {}) => {
|
||||
const {
|
||||
headers = {
|
||||
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0',
|
||||
},
|
||||
proxy = null,
|
||||
agent = null,
|
||||
signal = null,
|
||||
} = options
|
||||
|
||||
const res = proxy ? await profetch(url, { proxy, signal }) : await fetch(url, { headers, agent, signal })
|
||||
|
||||
const status = res.status
|
||||
if (status >= 400) {
|
||||
throw new Error(`Request failed with error code ${status}`)
|
||||
}
|
||||
const contentType = res.headers.get('content-type')
|
||||
const text = await res.text()
|
||||
|
||||
if (/(\+|\/)(xml|html)/.test(contentType)) {
|
||||
return { type: 'xml', text: text.trim(), status, contentType }
|
||||
}
|
||||
|
||||
if (/(\+|\/)json/.test(contentType)) {
|
||||
try {
|
||||
const data = JSON.parse(text)
|
||||
return { type: 'json', json: data, status, contentType }
|
||||
} catch (err) {
|
||||
throw new Error('Failed to convert data to JSON object')
|
||||
}
|
||||
}
|
||||
throw new Error(`Invalid content type: ${contentType}`)
|
||||
}
|
||||
-76
@@ -1,76 +0,0 @@
|
||||
// retrieve.test
|
||||
/* eslint-env jest */
|
||||
|
||||
import nock from 'nock'
|
||||
|
||||
import retrieve from './retrieve.js'
|
||||
|
||||
const parseUrl = (url) => {
|
||||
const re = new URL(url)
|
||||
return {
|
||||
baseUrl: `${re.protocol}//${re.host}`,
|
||||
path: re.pathname,
|
||||
}
|
||||
}
|
||||
|
||||
describe('test retrieve() method', () => {
|
||||
test('test retrieve with bad status code', async () => {
|
||||
const url = 'https://some.where/bad/page'
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(500, 'Error 500')
|
||||
expect(retrieve(url)).rejects.toThrow(new Error('Request failed with error code 500'))
|
||||
})
|
||||
|
||||
test('test retrieve with bad conten type', async () => {
|
||||
const url = 'https://some.where/bad/page'
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, '<?xml version="1.0"?><tag>this is xml</tag>', {
|
||||
'Content-Type': 'something/type',
|
||||
})
|
||||
expect(retrieve(url)).rejects.toThrow(new Error('Invalid content type: something/type'))
|
||||
})
|
||||
|
||||
test('test retrieve from good source', async () => {
|
||||
const url = 'https://some.where/good/page'
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, '<div>this is content</div>', {
|
||||
'Content-Type': 'application/rss+xml',
|
||||
})
|
||||
const result = await retrieve(url)
|
||||
expect(result.type).toEqual('xml')
|
||||
expect(result.text).toEqual('<div>this is content</div>')
|
||||
})
|
||||
|
||||
test('test retrieve from good source, but having \\r\\n before/after root xml', async () => {
|
||||
const url = 'https://some.where/good/page'
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, '\n\r\r\n\n<div>this is content</div>\n\r\r\n\n', {
|
||||
'Content-Type': 'text/xml',
|
||||
})
|
||||
const result = await retrieve(url)
|
||||
expect(result.type).toEqual('xml')
|
||||
expect(result.text).toBe('<div>this is content</div>')
|
||||
})
|
||||
|
||||
test('test retrieve using proxy', async () => {
|
||||
const url = 'https://some.where/good/source-with-proxy'
|
||||
const { baseUrl, path } = parseUrl(url)
|
||||
nock(baseUrl).get(path).reply(200, 'something bad', {
|
||||
'Content-Type': 'bad/thing',
|
||||
})
|
||||
nock('https://proxy-server.com')
|
||||
.get('/api/proxy?url=https%3A%2F%2Fsome.where%2Fgood%2Fsource-with-proxy')
|
||||
.reply(200, '<?xml version="1.0"?><tag>this is xml</tag>', {
|
||||
'Content-Type': 'text/xml',
|
||||
})
|
||||
|
||||
const result = await retrieve(url, {
|
||||
proxy: {
|
||||
target: 'https://proxy-server.com/api/proxy?url=',
|
||||
},
|
||||
})
|
||||
expect(result.type).toEqual('xml')
|
||||
expect(result.text).toEqual('<?xml version="1.0"?><tag>this is xml</tag>')
|
||||
nock.cleanAll()
|
||||
})
|
||||
})
|
||||
-32
@@ -1,32 +0,0 @@
|
||||
// utils / xmlparser
|
||||
|
||||
import { hasProperty, isString } from 'bellajs'
|
||||
|
||||
import { XMLValidator, XMLParser } from 'fast-xml-parser'
|
||||
|
||||
export const isRSS = (data = {}) => {
|
||||
return hasProperty(data, 'rss') && hasProperty(data.rss, 'channel')
|
||||
}
|
||||
|
||||
export const isAtom = (data = {}) => {
|
||||
return hasProperty(data, 'feed') && hasProperty(data.feed, 'entry')
|
||||
}
|
||||
|
||||
export const isRdf = (data = {}) => {
|
||||
return hasProperty(data, 'rdf:RDF') && hasProperty(data['rdf:RDF'], 'channel')
|
||||
}
|
||||
|
||||
export const validate = (xml) => {
|
||||
return (!isString(xml) || !xml.length) ? false : XMLValidator.validate(xml) === true
|
||||
}
|
||||
|
||||
export const xml2obj = (xml = '', extraOptions = {}) => {
|
||||
const options = {
|
||||
attributeNamePrefix: '@_',
|
||||
ignoreAttributes: false,
|
||||
...extraOptions,
|
||||
}
|
||||
const parser = new XMLParser(options)
|
||||
const jsonObj = parser.parse(xml)
|
||||
return jsonObj
|
||||
}
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
// xmlparser.test
|
||||
/* eslint-env jest */
|
||||
|
||||
import { readFileSync } from 'fs'
|
||||
|
||||
import { validate, isRSS, isAtom, xml2obj } from './xmlparser.js'
|
||||
|
||||
describe('test methods from `xmlparser`', () => {
|
||||
test('test validate(well format xml)', async () => {
|
||||
const xmlData = '<xml><atag id="12">value</atag></xml>'
|
||||
const result = validate(xmlData)
|
||||
expect(result).toBe(true)
|
||||
})
|
||||
|
||||
test('test validate(bad format xml)', async () => {
|
||||
const xmlData = '<xml><atag id="12">value</btag></xml>'
|
||||
const result = validate(xmlData)
|
||||
expect(result).toBe(false)
|
||||
})
|
||||
|
||||
test('test validate(standard rss content)', async () => {
|
||||
const xml = readFileSync('test-data/rss-feed-standard.xml', 'utf8')
|
||||
const xmlData = xml2obj(xml)
|
||||
expect(isRSS(xmlData)).toBe(true)
|
||||
expect(isAtom(xmlData)).toBe(false)
|
||||
})
|
||||
|
||||
test('test validate(standard atom content)', async () => {
|
||||
const xml = readFileSync('test-data/atom-feed-standard.xml', 'utf8')
|
||||
const xmlData = xml2obj(xml)
|
||||
expect(isAtom(xmlData)).toBe(true)
|
||||
expect(isRSS(xmlData)).toBe(false)
|
||||
})
|
||||
|
||||
test('test xml2obj(well format xml)', async () => {
|
||||
const xmlData = '<xml><atag id="12">value</atag></xml>'
|
||||
const result = xml2obj(xmlData)
|
||||
expect(result).toBeInstanceOf(Object)
|
||||
expect(result.xml).toBeInstanceOf(Object)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user