src/frame/middleware/index.ts

import fs from 'fs'
import path from 'path'

import express from 'express'
import type { NextFunction, Request, Response, Express } from 'express'
import timeout from 'connect-timeout'

import { haltOnDroppedConnection } from './halt-on-dropped-connection'
import abort from './abort'
import helmet from './helmet'
import cookieParser from './cookie-parser'
import {
  setDefaultFastlySurrogateKey,
  setLanguageFastlySurrogateKey,
} from './set-fastly-surrogate-key'
import handleErrors from '@/observability/middleware/handle-errors'
import handleNextDataPath from './handle-next-data-path'
import detectLanguage from '@/languages/middleware/detect-language'
import reloadTree from './reload-tree'
import context from './context/context'
import shortVersions from '@/versions/middleware/short-versions'
import languageCodeRedirects from '@/redirects/middleware/language-code-redirects'
import handleRedirects from '@/redirects/middleware/handle-redirects'
import findPage from './find-page'
import blockRobots from './block-robots'
import archivedEnterpriseVersionsAssets from '@/archives/middleware/archived-enterprise-versions-assets'
import api from './api'
import llmsTxt from './llms-txt'
import healthcheck from './healthcheck'
import manifestJson from './manifest-json'
import buildInfo from './build-info'
import reqHeaders from './req-headers'
import archivedEnterpriseVersions from '@/archives/middleware/archived-enterprise-versions'
import robots from './robots'
import earlyAccessLinks from '@/early-access/middleware/early-access-links'
import categoriesForSupport from './categories-for-support'
import triggerError from '@/observability/middleware/trigger-error'
import dataTables from '@/data-directory/middleware/data-tables'
import secretScanning from '@/secret-scanning/middleware/secret-scanning'
import ghesReleaseNotes from '@/release-notes/middleware/ghes-release-notes'
import whatsNewChangelog from './context/whats-new-changelog'
import layout from './context/layout'
import currentProductTree from './context/current-product-tree'
import genericToc from './context/generic-toc'
import breadcrumbs from './context/breadcrumbs'
import glossaries from './context/glossaries'
import resolveRecommended from './resolve-recommended'
import renderProductName from './context/render-product-name'
import features from '@/versions/middleware/features'
import productExamples from './context/product-examples'
import productGroups from './context/product-groups'
import featuredLinks from '@/landings/middleware/featured-links'
import learningTrack from '@/learning-track/middleware/learning-track'
import journeyTrack from '@/journeys/middleware/journey-track'
import next from './next'
import renderPage from './render-page'
import assetPreprocessing from '@/assets/middleware/asset-preprocessing'
import archivedAssetRedirects from '@/archives/middleware/archived-asset-redirects'
import favicons from './favicons'
import setStaticAssetCaching from '@/assets/middleware/static-asset-caching'
import fastHead from './fast-head'
import fastlyCacheTest from './fastly-cache-test'
import trailingSlashes from './trailing-slashes'
import mockVaPortal from './mock-va-portal'
import dynamicAssets from '@/assets/middleware/dynamic-assets'
import generalSearchMiddleware from '@/search/middleware/general-search-middleware'
import shielding from '@/shielding/middleware'
import { MAX_REQUEST_TIMEOUT } from '@/frame/lib/constants'
import { initLoggerContext } from '@/observability/logger/lib/logger-context'
import { getAutomaticRequestLogger } from '@/observability/logger/middleware/get-automatic-request-logger'
import appRouterGateway from './app-router-gateway'
import urlDecode from './url-decode'

const { NODE_ENV } = process.env
const isTest = NODE_ENV === 'test' || process.env.GITHUB_ACTIONS === 'true'

const ENABLE_FASTLY_TESTING = JSON.parse(process.env.ENABLE_FASTLY_TESTING || 'false')

// Catch unhandled promise rejections and passing them to Express's error handler
// https://medium.com/@Abazhenov/using-async-await-in-express-with-node-8-b8af872c0016
const asyncMiddleware =
  <TReq extends Request = Request, T = void>(
    fn: (req: TReq, res: Response, next: NextFunction) => T | Promise<T>,
  ) =>
  async (req: Request, res: Response, nextFn: NextFunction) => {
    try {
      await fn(req as TReq, res, nextFn)
    } catch (error) {
      nextFn(error)
    }
  }

export default function index(app: Express) {
  // *** Request connection management ***
  if (!isTest) app.use(timeout(MAX_REQUEST_TIMEOUT))
  app.use(abort)

  // Don't use the proxy's IP, use the requester's for rate limiting or
  // logging.
  // See https://expressjs.com/en/guide/behind-proxies.html
  // Essentially, setting this means it believe that the IP is the
  // first of the `X-Forwarded-For` header values.
  // If it was 0 (or false), the value would be that
  // of `req.socket.remoteAddress`.
  // Now, the `req.ip` becomes the first entry from x-forwarded-for
  // and falls back on `req.socket.remoteAddress` in all other cases.
  // Their documentation says:
  //
  //   	If true, the client's IP address is understood as the
  //    left-most entry in the X-Forwarded-For header.
  //
  app.set('trust proxy', true)

  // *** Logging ***
  app.use(initLoggerContext) // Context for both inline logs (e.g. logger.info) and automatic logs
  app.use(getAutomaticRequestLogger()) // Automatic logging for all requests e.g. "GET /path 200"

  // Put this early to make it as fast as possible because it's used
  // to check the health of each cluster.
  app.use('/healthcheck', healthcheck)

  // Must appear before static assets and all other requests
  // otherwise we won't be able to benefit from that functionality
  // for static assets as well.
  app.use(setDefaultFastlySurrogateKey)

  // archivedEnterpriseVersionsAssets must come before static/assets
  app.use(asyncMiddleware(archivedEnterpriseVersionsAssets))

  app.use(favicons)

  // Any static URL that contains some sort of checksum that makes it
  // unique gets the "manual" surrogate key. If it's checksummed,
  // it's bound to change when it needs to change. Otherwise,
  // we want to make sure it doesn't need to be purged just because
  // there's a production deploy.
  // Note, for `/assets/cb-*...` requests,
  // this needs to come before `assetPreprocessing` because
  // the `assetPreprocessing` middleware will rewrite `req.url` if
  // it applies.
  app.use(setStaticAssetCaching)

  // Must come before any other middleware for assets
  app.use(archivedAssetRedirects)

  // This must come before the express.static('assets') middleware.
  app.use(assetPreprocessing)

  app.use(
    '/assets/',
    express.static('assets', {
      index: false,
      etag: false,
      // Can be aggressive because images inside the content get unique
      // URLs with a cache busting prefix.
      maxAge: '7 days',
      immutable: process.env.NODE_ENV !== 'development',
      // The next middleware will try its luck and send the 404 if must.
      fallthrough: true,
    }),
  )
  app.use(asyncMiddleware(dynamicAssets))
  app.use(
    '/public/',
    express.static('src/graphql/data', {
      index: false,
      etag: false,
      maxAge: '7 days', // A bit longer since releases are more sparse
      // See note about the use of 'fallthrough'
      fallthrough: false,
    }),
  )

  // In development, let NextJS on-the-fly serve the static assets.
  // But in production, don't let NextJS handle any static assets
  // because they are costly to generate (the 404 HTML page).
  if (process.env.NODE_ENV !== 'development') {
    const assetDir = path.join('.next', 'static')
    if (!fs.existsSync(assetDir))
      throw new Error(`${assetDir} directory has not been generated. Run 'npm run build' first.`)

    app.use(
      '/_next/static/',
      express.static(assetDir, {
        index: false,
        etag: false,
        maxAge: '365 days',
        immutable: true,
        // See note about the use of 'fallthrough'
        fallthrough: false,
      }),
    )
  }

  // *** Early exits ***
  app.use(shielding)
  app.use(handleNextDataPath)

  // *** Security ***
  app.use(helmet)
  app.use(cookieParser)
  app.use(express.json())

  if (process.env.NODE_ENV === 'development') {
    app.use(mockVaPortal) // FOR TESTING.
  }

  // ** Possible early exits after cookies **

  // *** Headers ***
  app.set('etag', false) // We will manage our own ETags if desired

  // *** Config and context for redirects ***
  app.use(urlDecode) // Must come before detectLanguage to decode @ symbols in version segments
  app.use(detectLanguage) // Must come before context, breadcrumbs, find-page, handle-errors, homepages
  app.use(asyncMiddleware(reloadTree)) // Must come before context
  app.use(asyncMiddleware(context)) // Must come before early-access-*, handle-redirects
  app.use(shortVersions) // Support version shorthands
  app.use(asyncMiddleware(renderProductName)) // Must come after shortVersions

  // Must come before handleRedirects.
  // This middleware might either redirect to serve something.
  app.use(asyncMiddleware(archivedEnterpriseVersions))

  // *** Redirects, 3xx responses ***
  // I ordered these by use frequency
  app.use(trailingSlashes)
  app.use(languageCodeRedirects) // Must come before contextualizers
  app.use(handleRedirects) // Must come before contextualizers

  // *** Config and context for rendering ***
  app.use(asyncMiddleware(findPage)) // Must come before archived-enterprise-versions, breadcrumbs, featured-links, products, render-page
  app.use(blockRobots)

  // Check for a dropped connection before proceeding
  app.use(haltOnDroppedConnection)

  // *** Add App Router Gateway here - before heavy contextualizers ***
  app.use(asyncMiddleware(appRouterGateway))

  // *** Rendering, 2xx responses ***
  app.use('/api', api)
  app.use('/llms.txt', llmsTxt)
  app.get('/_build', buildInfo)
  app.get('/_req-headers', reqHeaders)
  app.use(asyncMiddleware(manifestJson))

  // Things like `/api` sets their own Fastly surrogate keys.
  // Now that the `req.language` is known, set it for the remaining endpoints
  app.use(setLanguageFastlySurrogateKey)

  // Check for a dropped connection before proceeding (again)
  app.use(haltOnDroppedConnection)

  app.use(robots)
  app.use(earlyAccessLinks)
  app.use('/categories.json', asyncMiddleware(categoriesForSupport))
  app.get('/_500', asyncMiddleware(triggerError))

  // Check for a dropped connection before proceeding (again)
  app.use(haltOnDroppedConnection)

  // Specifically deal with HEAD requests before doing the slower
  // full page rendering.
  app.head('/*path', fastHead)

  // *** Preparation for render-page: contextualizers ***
  app.use(asyncMiddleware(dataTables))
  app.use(asyncMiddleware(secretScanning))
  app.use(asyncMiddleware(ghesReleaseNotes))
  app.use(asyncMiddleware(whatsNewChangelog))
  app.use(layout)
  app.use(features) // needs to come before product tree
  app.use(asyncMiddleware(currentProductTree))
  app.use(asyncMiddleware(genericToc))
  app.use(breadcrumbs)
  app.use(asyncMiddleware(productExamples))
  app.use(asyncMiddleware(productGroups))
  app.use(asyncMiddleware(glossaries))
  app.use(asyncMiddleware(generalSearchMiddleware))
  app.use(asyncMiddleware(featuredLinks))
  app.use(asyncMiddleware(resolveRecommended))
  app.use(asyncMiddleware(learningTrack))
  app.use(asyncMiddleware(journeyTrack))

  if (ENABLE_FASTLY_TESTING) {
    // The fastlyCacheTest middleware is intended to be used with Fastly to test caching behavior.
    // This middleware will intercept ALL requests routed to it, so be careful if you need to
    // make any changes to the following line:
    app.use('/fastly-cache-test', fastlyCacheTest)
  }

  // handle serving NextJS bundled code (/_next/*)
  app.use(next)

  // Check for a dropped connection before proceeding (again)
  app.use(haltOnDroppedConnection)

  // *** Rendering, must go almost last ***
  app.get('/*path', asyncMiddleware(renderPage))

  // *** Error handling, must go last ***
  app.use(handleErrors)
}