working rss feed to nostr publish

This commit is contained in:
2023-11-24 00:43:28 -05:00
parent 06edcb57ae
commit 88d2f9cfec
8396 changed files with 783105 additions and 0 deletions
+28
View File
@@ -0,0 +1,28 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.AdaptiveRetryStrategy = void 0;
const config_1 = require("./config");
const DefaultRateLimiter_1 = require("./DefaultRateLimiter");
const StandardRetryStrategy_1 = require("./StandardRetryStrategy");
class AdaptiveRetryStrategy {
constructor(maxAttemptsProvider, options) {
this.maxAttemptsProvider = maxAttemptsProvider;
this.mode = config_1.RETRY_MODES.ADAPTIVE;
const { rateLimiter } = options !== null && options !== void 0 ? options : {};
this.rateLimiter = rateLimiter !== null && rateLimiter !== void 0 ? rateLimiter : new DefaultRateLimiter_1.DefaultRateLimiter();
this.standardRetryStrategy = new StandardRetryStrategy_1.StandardRetryStrategy(maxAttemptsProvider);
}
async acquireInitialRetryToken(retryTokenScope) {
await this.rateLimiter.getSendToken();
return this.standardRetryStrategy.acquireInitialRetryToken(retryTokenScope);
}
async refreshRetryTokenForRetry(tokenToRenew, errorInfo) {
this.rateLimiter.updateClientSendingRate(errorInfo);
return this.standardRetryStrategy.refreshRetryTokenForRetry(tokenToRenew, errorInfo);
}
recordSuccess(token) {
this.rateLimiter.updateClientSendingRate({});
this.standardRetryStrategy.recordSuccess(token);
}
}
exports.AdaptiveRetryStrategy = AdaptiveRetryStrategy;
+22
View File
@@ -0,0 +1,22 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ConfiguredRetryStrategy = void 0;
const constants_1 = require("./constants");
const StandardRetryStrategy_1 = require("./StandardRetryStrategy");
class ConfiguredRetryStrategy extends StandardRetryStrategy_1.StandardRetryStrategy {
constructor(maxAttempts, computeNextBackoffDelay = constants_1.DEFAULT_RETRY_DELAY_BASE) {
super(typeof maxAttempts === "function" ? maxAttempts : async () => maxAttempts);
if (typeof computeNextBackoffDelay === "number") {
this.computeNextBackoffDelay = () => computeNextBackoffDelay;
}
else {
this.computeNextBackoffDelay = computeNextBackoffDelay;
}
}
async refreshRetryTokenForRetry(tokenToRenew, errorInfo) {
const token = await super.refreshRetryTokenForRetry(tokenToRenew, errorInfo);
token.getRetryDelay = () => this.computeNextBackoffDelay(token.getRetryCount());
return token;
}
}
exports.ConfiguredRetryStrategy = ConfiguredRetryStrategy;
+104
View File
@@ -0,0 +1,104 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.DefaultRateLimiter = void 0;
const service_error_classification_1 = require("@smithy/service-error-classification");
class DefaultRateLimiter {
constructor(options) {
var _a, _b, _c, _d, _e;
this.currentCapacity = 0;
this.enabled = false;
this.lastMaxRate = 0;
this.measuredTxRate = 0;
this.requestCount = 0;
this.lastTimestamp = 0;
this.timeWindow = 0;
this.beta = (_a = options === null || options === void 0 ? void 0 : options.beta) !== null && _a !== void 0 ? _a : 0.7;
this.minCapacity = (_b = options === null || options === void 0 ? void 0 : options.minCapacity) !== null && _b !== void 0 ? _b : 1;
this.minFillRate = (_c = options === null || options === void 0 ? void 0 : options.minFillRate) !== null && _c !== void 0 ? _c : 0.5;
this.scaleConstant = (_d = options === null || options === void 0 ? void 0 : options.scaleConstant) !== null && _d !== void 0 ? _d : 0.4;
this.smooth = (_e = options === null || options === void 0 ? void 0 : options.smooth) !== null && _e !== void 0 ? _e : 0.8;
const currentTimeInSeconds = this.getCurrentTimeInSeconds();
this.lastThrottleTime = currentTimeInSeconds;
this.lastTxRateBucket = Math.floor(this.getCurrentTimeInSeconds());
this.fillRate = this.minFillRate;
this.maxCapacity = this.minCapacity;
}
getCurrentTimeInSeconds() {
return Date.now() / 1000;
}
async getSendToken() {
return this.acquireTokenBucket(1);
}
async acquireTokenBucket(amount) {
if (!this.enabled) {
return;
}
this.refillTokenBucket();
if (amount > this.currentCapacity) {
const delay = ((amount - this.currentCapacity) / this.fillRate) * 1000;
await new Promise((resolve) => setTimeout(resolve, delay));
}
this.currentCapacity = this.currentCapacity - amount;
}
refillTokenBucket() {
const timestamp = this.getCurrentTimeInSeconds();
if (!this.lastTimestamp) {
this.lastTimestamp = timestamp;
return;
}
const fillAmount = (timestamp - this.lastTimestamp) * this.fillRate;
this.currentCapacity = Math.min(this.maxCapacity, this.currentCapacity + fillAmount);
this.lastTimestamp = timestamp;
}
updateClientSendingRate(response) {
let calculatedRate;
this.updateMeasuredRate();
if ((0, service_error_classification_1.isThrottlingError)(response)) {
const rateToUse = !this.enabled ? this.measuredTxRate : Math.min(this.measuredTxRate, this.fillRate);
this.lastMaxRate = rateToUse;
this.calculateTimeWindow();
this.lastThrottleTime = this.getCurrentTimeInSeconds();
calculatedRate = this.cubicThrottle(rateToUse);
this.enableTokenBucket();
}
else {
this.calculateTimeWindow();
calculatedRate = this.cubicSuccess(this.getCurrentTimeInSeconds());
}
const newRate = Math.min(calculatedRate, 2 * this.measuredTxRate);
this.updateTokenBucketRate(newRate);
}
calculateTimeWindow() {
this.timeWindow = this.getPrecise(Math.pow((this.lastMaxRate * (1 - this.beta)) / this.scaleConstant, 1 / 3));
}
cubicThrottle(rateToUse) {
return this.getPrecise(rateToUse * this.beta);
}
cubicSuccess(timestamp) {
return this.getPrecise(this.scaleConstant * Math.pow(timestamp - this.lastThrottleTime - this.timeWindow, 3) + this.lastMaxRate);
}
enableTokenBucket() {
this.enabled = true;
}
updateTokenBucketRate(newRate) {
this.refillTokenBucket();
this.fillRate = Math.max(newRate, this.minFillRate);
this.maxCapacity = Math.max(newRate, this.minCapacity);
this.currentCapacity = Math.min(this.currentCapacity, this.maxCapacity);
}
updateMeasuredRate() {
const t = this.getCurrentTimeInSeconds();
const timeBucket = Math.floor(t * 2) / 2;
this.requestCount++;
if (timeBucket > this.lastTxRateBucket) {
const currentRate = this.requestCount / (timeBucket - this.lastTxRateBucket);
this.measuredTxRate = this.getPrecise(currentRate * this.smooth + this.measuredTxRate * (1 - this.smooth));
this.requestCount = 0;
this.lastTxRateBucket = timeBucket;
}
}
getPrecise(num) {
return parseFloat(num.toFixed(8));
}
}
exports.DefaultRateLimiter = DefaultRateLimiter;
+70
View File
@@ -0,0 +1,70 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.StandardRetryStrategy = void 0;
const config_1 = require("./config");
const constants_1 = require("./constants");
const defaultRetryBackoffStrategy_1 = require("./defaultRetryBackoffStrategy");
const defaultRetryToken_1 = require("./defaultRetryToken");
class StandardRetryStrategy {
constructor(maxAttempts) {
this.maxAttempts = maxAttempts;
this.mode = config_1.RETRY_MODES.STANDARD;
this.capacity = constants_1.INITIAL_RETRY_TOKENS;
this.retryBackoffStrategy = (0, defaultRetryBackoffStrategy_1.getDefaultRetryBackoffStrategy)();
this.maxAttemptsProvider = typeof maxAttempts === "function" ? maxAttempts : async () => maxAttempts;
}
async acquireInitialRetryToken(retryTokenScope) {
return (0, defaultRetryToken_1.createDefaultRetryToken)({
retryDelay: constants_1.DEFAULT_RETRY_DELAY_BASE,
retryCount: 0,
});
}
async refreshRetryTokenForRetry(token, errorInfo) {
const maxAttempts = await this.getMaxAttempts();
if (this.shouldRetry(token, errorInfo, maxAttempts)) {
const errorType = errorInfo.errorType;
this.retryBackoffStrategy.setDelayBase(errorType === "THROTTLING" ? constants_1.THROTTLING_RETRY_DELAY_BASE : constants_1.DEFAULT_RETRY_DELAY_BASE);
const delayFromErrorType = this.retryBackoffStrategy.computeNextBackoffDelay(token.getRetryCount());
const retryDelay = errorInfo.retryAfterHint
? Math.max(errorInfo.retryAfterHint.getTime() - Date.now() || 0, delayFromErrorType)
: delayFromErrorType;
const capacityCost = this.getCapacityCost(errorType);
this.capacity -= capacityCost;
return (0, defaultRetryToken_1.createDefaultRetryToken)({
retryDelay,
retryCount: token.getRetryCount() + 1,
retryCost: capacityCost,
});
}
throw new Error("No retry token available");
}
recordSuccess(token) {
var _a;
this.capacity = Math.max(constants_1.INITIAL_RETRY_TOKENS, this.capacity + ((_a = token.getRetryCost()) !== null && _a !== void 0 ? _a : constants_1.NO_RETRY_INCREMENT));
}
getCapacity() {
return this.capacity;
}
async getMaxAttempts() {
try {
return await this.maxAttemptsProvider();
}
catch (error) {
console.warn(`Max attempts provider could not resolve. Using default of ${config_1.DEFAULT_MAX_ATTEMPTS}`);
return config_1.DEFAULT_MAX_ATTEMPTS;
}
}
shouldRetry(tokenToRenew, errorInfo, maxAttempts) {
const attempts = tokenToRenew.getRetryCount() + 1;
return (attempts < maxAttempts &&
this.capacity >= this.getCapacityCost(errorInfo.errorType) &&
this.isRetryableError(errorInfo.errorType));
}
getCapacityCost(errorType) {
return errorType === "TRANSIENT" ? constants_1.TIMEOUT_RETRY_COST : constants_1.RETRY_COST;
}
isRetryableError(errorType) {
return errorType === "THROTTLING" || errorType === "TRANSIENT";
}
}
exports.StandardRetryStrategy = StandardRetryStrategy;
+10
View File
@@ -0,0 +1,10 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.DEFAULT_RETRY_MODE = exports.DEFAULT_MAX_ATTEMPTS = exports.RETRY_MODES = void 0;
var RETRY_MODES;
(function (RETRY_MODES) {
RETRY_MODES["STANDARD"] = "standard";
RETRY_MODES["ADAPTIVE"] = "adaptive";
})(RETRY_MODES = exports.RETRY_MODES || (exports.RETRY_MODES = {}));
exports.DEFAULT_MAX_ATTEMPTS = 3;
exports.DEFAULT_RETRY_MODE = RETRY_MODES.STANDARD;
+12
View File
@@ -0,0 +1,12 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.REQUEST_HEADER = exports.INVOCATION_ID_HEADER = exports.NO_RETRY_INCREMENT = exports.TIMEOUT_RETRY_COST = exports.RETRY_COST = exports.INITIAL_RETRY_TOKENS = exports.THROTTLING_RETRY_DELAY_BASE = exports.MAXIMUM_RETRY_DELAY = exports.DEFAULT_RETRY_DELAY_BASE = void 0;
exports.DEFAULT_RETRY_DELAY_BASE = 100;
exports.MAXIMUM_RETRY_DELAY = 20 * 1000;
exports.THROTTLING_RETRY_DELAY_BASE = 500;
exports.INITIAL_RETRY_TOKENS = 500;
exports.RETRY_COST = 5;
exports.TIMEOUT_RETRY_COST = 10;
exports.NO_RETRY_INCREMENT = 1;
exports.INVOCATION_ID_HEADER = "amz-sdk-invocation-id";
exports.REQUEST_HEADER = "amz-sdk-request";
@@ -0,0 +1,18 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.getDefaultRetryBackoffStrategy = void 0;
const constants_1 = require("./constants");
const getDefaultRetryBackoffStrategy = () => {
let delayBase = constants_1.DEFAULT_RETRY_DELAY_BASE;
const computeNextBackoffDelay = (attempts) => {
return Math.floor(Math.min(constants_1.MAXIMUM_RETRY_DELAY, Math.random() * 2 ** attempts * delayBase));
};
const setDelayBase = (delay) => {
delayBase = delay;
};
return {
computeNextBackoffDelay,
setDelayBase,
};
};
exports.getDefaultRetryBackoffStrategy = getDefaultRetryBackoffStrategy;
+15
View File
@@ -0,0 +1,15 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.createDefaultRetryToken = void 0;
const constants_1 = require("./constants");
const createDefaultRetryToken = ({ retryDelay, retryCount, retryCost, }) => {
const getRetryCount = () => retryCount;
const getRetryDelay = () => Math.min(constants_1.MAXIMUM_RETRY_DELAY, retryDelay);
const getRetryCost = () => retryCost;
return {
getRetryCount,
getRetryDelay,
getRetryCost,
};
};
exports.createDefaultRetryToken = createDefaultRetryToken;
+10
View File
@@ -0,0 +1,10 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("tslib");
tslib_1.__exportStar(require("./AdaptiveRetryStrategy"), exports);
tslib_1.__exportStar(require("./ConfiguredRetryStrategy"), exports);
tslib_1.__exportStar(require("./DefaultRateLimiter"), exports);
tslib_1.__exportStar(require("./StandardRetryStrategy"), exports);
tslib_1.__exportStar(require("./config"), exports);
tslib_1.__exportStar(require("./constants"), exports);
tslib_1.__exportStar(require("./types"), exports);
+2
View File
@@ -0,0 +1,2 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });