build/stand-alone/crawl.js

"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.CrawlStandAloneServer = void 0;
require("reflect-metadata");
const tsyringe_1 = require("tsyringe");
const koa_1 = require("civkit/civ-rpc/koa");
const http2_1 = __importDefault(require("http2"));
const crawler_1 = require("../api/crawler");
const fswalk_1 = require("civkit/fswalk");
const path_1 = __importDefault(require("path"));
const fs_1 = __importDefault(require("fs"));
const mime_1 = require("civkit/mime");
const registry_1 = require("../services/registry");
const async_hooks_1 = require("async_hooks");
const decorators_1 = require("civkit/decorators");
const crypto_1 = require("crypto");
const threaded_1 = require("../services/threaded");
const logger_1 = require("../services/logger");
const async_context_1 = require("../services/async-context");
const finalizer_1 = __importStar(require("../services/finalizer"));
const koa_compress_1 = __importDefault(require("koa-compress"));
let CrawlStandAloneServer = class CrawlStandAloneServer extends koa_1.KoaServer {
    constructor(globalLogger, registry, crawlerHost, threadLocal, threads) {
        super(...arguments);
        this.globalLogger = globalLogger;
        this.registry = registry;
        this.crawlerHost = crawlerHost;
        this.threadLocal = threadLocal;
        this.threads = threads;
        this.logger = this.globalLogger.child({ service: this.constructor.name });
        this.assets = new Map();
    }
    h2c() {
        this.httpAlternativeServer = this.httpServer;
        const fn = this.koaApp.callback();
        this.httpServer = http2_1.default.createServer((req, res) => {
            const ar = new async_hooks_1.AsyncResource('HTTP2ServerRequest');
            ar.runInAsyncScope(fn, this.koaApp, req, res);
        });
        // useResourceBasedDefaultTracker();
        return this;
    }
    async init() {
        await this.walkForAssets();
        await super.init();
    }
    async walkForAssets() {
        const files = await fswalk_1.FsWalk.walkOut(path_1.default.resolve(__dirname, '..', '..', 'public'));
        for (const file of files) {
            if (file.type !== 'file') {
                continue;
            }
            this.assets.set(file.relativePath.toString(), file);
        }
    }
    listen(port) {
        const r = super.listen(port);
        if (this.httpAlternativeServer) {
            const altPort = port + 1;
            this.httpAlternativeServer.listen(altPort, () => {
                this.logger.info(`Alternative ${this.httpAlternativeServer.constructor.name} listening on port ${altPort}`);
            });
        }
        return r;
    }
    makeAssetsServingController() {
        return (ctx, next) => {
            const requestPath = ctx.path;
            const file = requestPath.slice(1);
            if (!file) {
                return next();
            }
            const asset = this.assets.get(file);
            if (asset?.type !== 'file') {
                return next();
            }
            ctx.body = fs_1.default.createReadStream(asset.path);
            ctx.type = (0, mime_1.mimeOfExt)(path_1.default.extname(asset.path.toString())) || 'application/octet-stream';
            ctx.set('Content-Length', asset.stats.size.toString());
            return;
        };
    }
    registerRoutes() {
        this.koaApp.use((0, koa_compress_1.default)({
            filter(type) {
                if (type.startsWith('text/')) {
                    return true;
                }
                if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {
                    return true;
                }
                if (type.includes('application/x-ndjson')) {
                    return true;
                }
                return false;
            }
        }));
        this.koaApp.use(this.makeAssetsServingController());
        this.koaApp.use(this.registry.makeShimController());
    }
    // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context
    // TraceId is expected to be request-bound and unique. So these two has to be distinguished.
    insertAsyncHookMiddleware() {
        const asyncHookMiddleware = async (ctx, next) => {
            const googleTraceId = ctx.get('x-cloud-trace-context').split('/')?.[0];
            this.threadLocal.setup({
                traceId: (0, crypto_1.randomUUID)(),
                traceT0: new Date(),
                googleTraceId,
            });
            return next();
        };
        this.koaApp.use(asyncHookMiddleware);
    }
    async standDown() {
        const tasks = [];
        if (this.httpAlternativeServer?.listening) {
            this.httpAlternativeServer.closeIdleConnections?.();
            this.httpAlternativeServer.close();
            tasks.push(new Promise((resolve, reject) => {
                this.httpAlternativeServer.close((err) => {
                    if (err) {
                        return reject(err);
                    }
                    resolve();
                });
            }));
        }
        tasks.push(super.standDown());
        await Promise.all(tasks);
    }
};
exports.CrawlStandAloneServer = CrawlStandAloneServer;
__decorate([
    (0, decorators_1.runOnce)(),
    __metadata("design:type", Function),
    __metadata("design:paramtypes", []),
    __metadata("design:returntype", void 0)
], CrawlStandAloneServer.prototype, "insertAsyncHookMiddleware", null);
__decorate([
    (0, finalizer_1.Finalizer)(),
    __metadata("design:type", Function),
    __metadata("design:paramtypes", []),
    __metadata("design:returntype", Promise)
], CrawlStandAloneServer.prototype, "standDown", null);
exports.CrawlStandAloneServer = CrawlStandAloneServer = __decorate([
    (0, tsyringe_1.singleton)(),
    __metadata("design:paramtypes", [logger_1.GlobalLogger,
        registry_1.RPCRegistry,
        crawler_1.CrawlerHost,
        async_context_1.AsyncLocalContext,
        threaded_1.ThreadedServiceRegistry])
], CrawlStandAloneServer);
const instance = tsyringe_1.container.resolve(CrawlStandAloneServer);
exports.default = instance;
if (process.env.NODE_ENV?.includes('dry-run')) {
    instance.serviceReady().then(() => finalizer_1.default.terminate());
}
else {
    instance.serviceReady().then((s) => s.h2c().listen(parseInt(process.env.PORT || '') || 3000));
}
//# sourceMappingURL=crawl.js.map