"use strict"; var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { if (k2 === undefined) k2 = k; var desc = Object.getOwnPropertyDescriptor(m, k); if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { desc = { enumerable: true, get: function() { return m[k]; } }; } Object.defineProperty(o, k2, desc); }) : (function(o, m, k, k2) { if (k2 === undefined) k2 = k; o[k2] = m[k]; })); var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { Object.defineProperty(o, "default", { enumerable: true, value: v }); }) : function(o, v) { o["default"] = v; }); var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) { var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d; if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc); else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r; return c > 3 && r && Object.defineProperty(target, key, r), r; }; var __importStar = (this && this.__importStar) || (function () { var ownKeys = function(o) { ownKeys = Object.getOwnPropertyNames || function (o) { var ar = []; for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k; return ar; }; return ownKeys(o); }; return function (mod) { if (mod && mod.__esModule) return mod; var result = {}; if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]); __setModuleDefault(result, mod); return result; }; })(); var __metadata = (this && this.__metadata) || function (k, v) { if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v); }; var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); exports.CrawlStandAloneServer = void 0; require("reflect-metadata"); const tsyringe_1 = require("tsyringe"); const koa_1 = require("civkit/civ-rpc/koa"); const http2_1 = __importDefault(require("http2")); const crawler_1 = require("../api/crawler"); const fswalk_1 = require("civkit/fswalk"); const path_1 = __importDefault(require("path")); const fs_1 = __importDefault(require("fs")); const mime_1 = require("civkit/mime"); const registry_1 = require("../services/registry"); const async_hooks_1 = require("async_hooks"); const decorators_1 = require("civkit/decorators"); const crypto_1 = require("crypto"); const threaded_1 = require("../services/threaded"); const logger_1 = require("../services/logger"); const async_context_1 = require("../services/async-context"); const finalizer_1 = __importStar(require("../services/finalizer")); const koa_compress_1 = __importDefault(require("koa-compress")); let CrawlStandAloneServer = class CrawlStandAloneServer extends koa_1.KoaServer { constructor(globalLogger, registry, crawlerHost, threadLocal, threads) { super(...arguments); this.globalLogger = globalLogger; this.registry = registry; this.crawlerHost = crawlerHost; this.threadLocal = threadLocal; this.threads = threads; this.logger = this.globalLogger.child({ service: this.constructor.name }); this.assets = new Map(); } h2c() { this.httpAlternativeServer = this.httpServer; const fn = this.koaApp.callback(); this.httpServer = http2_1.default.createServer((req, res) => { const ar = new async_hooks_1.AsyncResource('HTTP2ServerRequest'); ar.runInAsyncScope(fn, this.koaApp, req, res); }); // useResourceBasedDefaultTracker(); return this; } async init() { await this.walkForAssets(); await super.init(); } async walkForAssets() { const files = await fswalk_1.FsWalk.walkOut(path_1.default.resolve(__dirname, '..', '..', 'public')); for (const file of files) { if (file.type !== 'file') { continue; } this.assets.set(file.relativePath.toString(), file); } } listen(port) { const r = super.listen(port); if (this.httpAlternativeServer) { const altPort = port + 1; this.httpAlternativeServer.listen(altPort, () => { this.logger.info(`Alternative ${this.httpAlternativeServer.constructor.name} listening on port ${altPort}`); }); } return r; } makeAssetsServingController() { return (ctx, next) => { const requestPath = ctx.path; const file = requestPath.slice(1); if (!file) { return next(); } const asset = this.assets.get(file); if (asset?.type !== 'file') { return next(); } ctx.body = fs_1.default.createReadStream(asset.path); ctx.type = (0, mime_1.mimeOfExt)(path_1.default.extname(asset.path.toString())) || 'application/octet-stream'; ctx.set('Content-Length', asset.stats.size.toString()); return; }; } registerRoutes() { this.koaApp.use((0, koa_compress_1.default)({ filter(type) { if (type.startsWith('text/')) { return true; } if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) { return true; } if (type.includes('application/x-ndjson')) { return true; } return false; } })); this.koaApp.use(this.makeAssetsServingController()); this.koaApp.use(this.registry.makeShimController()); } // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context // TraceId is expected to be request-bound and unique. So these two has to be distinguished. insertAsyncHookMiddleware() { const asyncHookMiddleware = async (ctx, next) => { const googleTraceId = ctx.get('x-cloud-trace-context').split('/')?.[0]; this.threadLocal.setup({ traceId: (0, crypto_1.randomUUID)(), traceT0: new Date(), googleTraceId, }); return next(); }; this.koaApp.use(asyncHookMiddleware); } async standDown() { const tasks = []; if (this.httpAlternativeServer?.listening) { this.httpAlternativeServer.closeIdleConnections?.(); this.httpAlternativeServer.close(); tasks.push(new Promise((resolve, reject) => { this.httpAlternativeServer.close((err) => { if (err) { return reject(err); } resolve(); }); })); } tasks.push(super.standDown()); await Promise.all(tasks); } }; exports.CrawlStandAloneServer = CrawlStandAloneServer; __decorate([ (0, decorators_1.runOnce)(), __metadata("design:type", Function), __metadata("design:paramtypes", []), __metadata("design:returntype", void 0) ], CrawlStandAloneServer.prototype, "insertAsyncHookMiddleware", null); __decorate([ (0, finalizer_1.Finalizer)(), __metadata("design:type", Function), __metadata("design:paramtypes", []), __metadata("design:returntype", Promise) ], CrawlStandAloneServer.prototype, "standDown", null); exports.CrawlStandAloneServer = CrawlStandAloneServer = __decorate([ (0, tsyringe_1.singleton)(), __metadata("design:paramtypes", [logger_1.GlobalLogger, registry_1.RPCRegistry, crawler_1.CrawlerHost, async_context_1.AsyncLocalContext, threaded_1.ThreadedServiceRegistry]) ], CrawlStandAloneServer); const instance = tsyringe_1.container.resolve(CrawlStandAloneServer); exports.default = instance; if (process.env.NODE_ENV?.includes('dry-run')) { instance.serviceReady().then(() => finalizer_1.default.terminate()); } else { instance.serviceReady().then((s) => s.h2c().listen(parseInt(process.env.PORT || '') || 3000)); } //# sourceMappingURL=crawl.js.map