File size: 5,118 Bytes
aec3094 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | import { Logger } from '@n8n/backend-common';
import { ExecutionsConfig } from '@n8n/config';
import { ExecutionRepository } from '@n8n/db';
import { OnLeaderStepdown, OnLeaderTakeover, OnShutdown } from '@n8n/decorators';
import { Service } from '@n8n/di';
import { BinaryDataService, InstanceSettings } from 'n8n-core';
import { ensureError } from 'n8n-workflow';
import { strict } from 'node:assert';
import { Time } from '@/constants';
import { DbConnection } from '@/databases/db-connection';
/**
* Responsible for deleting old executions from the database and deleting their
* associated binary data from the filesystem, on a rolling basis.
*
* By default:
*
* - Soft deletion (every 60m) identifies all prunable executions based on max
* age and/or max count, exempting annotated executions.
* - Hard deletion (every 15m) processes prunable executions in batches of 100,
* switching to 1s intervals until the total to prune is back down low enough,
* or in case the hard deletion fails.
* - Once mostly caught up, hard deletion goes back to the 15m schedule.
*/
@Service()
export class ExecutionsPruningService {
/** Timer for soft-deleting executions on a rolling basis. */
private softDeletionInterval: NodeJS.Timeout | undefined;
/** Timeout for next hard-deletion of soft-deleted executions. */
private hardDeletionTimeout: NodeJS.Timeout | undefined;
private readonly rates = {
softDeletion: this.executionsConfig.pruneDataIntervals.softDelete * Time.minutes.toMilliseconds,
hardDeletion: this.executionsConfig.pruneDataIntervals.hardDelete * Time.minutes.toMilliseconds,
};
/** Max number of executions to hard-delete in a cycle. */
private readonly batchSize = 100;
private isShuttingDown = false;
constructor(
private readonly logger: Logger,
private readonly instanceSettings: InstanceSettings,
private readonly dbConnection: DbConnection,
private readonly executionRepository: ExecutionRepository,
private readonly binaryDataService: BinaryDataService,
private readonly executionsConfig: ExecutionsConfig,
) {
this.logger = this.logger.scoped('pruning');
}
init() {
strict(this.instanceSettings.instanceRole !== 'unset', 'Instance role is not set');
if (this.instanceSettings.isLeader) this.startPruning();
}
get isEnabled() {
return (
this.executionsConfig.pruneData &&
this.instanceSettings.instanceType === 'main' &&
this.instanceSettings.isLeader
);
}
@OnLeaderTakeover()
startPruning() {
const { connectionState } = this.dbConnection;
if (!this.isEnabled || !connectionState.migrated || this.isShuttingDown) return;
this.scheduleRollingSoftDeletions();
this.scheduleNextHardDeletion();
this.logger.debug('Started pruning timers');
}
@OnLeaderStepdown()
stopPruning() {
if (!this.isEnabled) return;
clearInterval(this.softDeletionInterval);
clearTimeout(this.hardDeletionTimeout);
this.logger.debug('Stopped pruning timers');
}
private scheduleRollingSoftDeletions(rateMs = this.rates.softDeletion) {
this.softDeletionInterval = setInterval(
async () => await this.softDelete(),
this.rates.softDeletion,
);
this.logger.debug(`Soft-deletion every ${rateMs * Time.milliseconds.toMinutes} minutes`);
}
private scheduleNextHardDeletion(rateMs = this.rates.hardDeletion) {
this.hardDeletionTimeout = setTimeout(() => {
this.hardDelete()
.then((rate) => this.scheduleNextHardDeletion(rate))
.catch((error) => {
this.scheduleNextHardDeletion(1_000);
this.logger.error('Failed to hard-delete executions', { error: ensureError(error) });
});
}, rateMs);
this.logger.debug(`Hard-deletion in next ${rateMs * Time.milliseconds.toMinutes} minutes`);
}
/** Soft-delete executions based on max age and/or max count. */
async softDelete() {
const result = await this.executionRepository.softDeletePrunableExecutions();
if (result.affected === 0) {
this.logger.debug('Found no executions to soft-delete');
return;
}
this.logger.debug('Soft-deleted executions', { count: result.affected });
}
@OnShutdown()
shutdown(): void {
this.isShuttingDown = true;
this.stopPruning();
}
/**
* Delete all soft-deleted executions and their binary data.
*
* @returns Delay in milliseconds until next hard-deletion
*/
private async hardDelete(): Promise<number> {
const ids = await this.executionRepository.findSoftDeletedExecutions();
const executionIds = ids.map((o) => o.executionId);
if (executionIds.length === 0) {
this.logger.debug('Found no executions to hard-delete');
return this.rates.hardDeletion;
}
try {
await this.binaryDataService.deleteMany(ids);
await this.executionRepository.deleteByIds(executionIds);
this.logger.debug('Hard-deleted executions', { executionIds });
} catch (error) {
this.logger.error('Failed to hard-delete executions', {
executionIds,
error: ensureError(error),
});
}
// if high volume, speed up next hard-deletion
if (executionIds.length >= this.batchSize) return 1 * Time.seconds.toMilliseconds;
return this.rates.hardDeletion;
}
}
|