src/content-linter/scripts/find-unsed-variables.ts

/**
 * @purpose Writer tool
 * @description Look for mentions of variables in Liquid syntax across all pages
 *
 * For example,
 *
 *    ---
 *    title: '{% data variables.product.prodname_mobile %} is cool'
 *    shortTitle: '{% data variables.product.prodname_mobile %}'
 *    ---
 *
 *    This also mentions {% data variables.product.prodname_ios %}
 *
 * So in this case, we *know* that `prodname_mobile` and
 * `prodname_ios` inside `data/variables/product.yml` is definitely used.
 * So that variable won't be mentioned as unused.
 *
 */
import fs from 'fs'
import yaml from 'js-yaml'

import { program } from 'commander'

import { loadPages, loadUnversionedTree } from '@/frame/lib/page-data'
import { TokenizationError, TokenKind } from 'liquidjs'
import type { TagToken } from 'liquidjs'

import readFrontmatter from '@/frame/lib/read-frontmatter'
import { getLiquidTokens } from '@/content-linter/lib/helpers/liquid-utils'
import walkFiles from '@/workflows/walk-files'

program
  .description('Finds unused variables in frontmatter, content, and reusables')
  .option('-o, --output-file <path>', 'path to output file', 'stdout')
  .option('--json', 'serialize output in JSON')
  .option('--markdown', 'serialize output as a Markdown comment')
  .parse(process.argv)

type Options = {
  outputFile: string
  json?: boolean
  markdown?: boolean
}
main(program.opts())

async function main(options: Options) {
  const variables = getVariables()
  const pages = await getPages()
  for (const page of pages) {
    try {
      const filePath = page.fullPath
      const fileContent = fs.readFileSync(filePath, 'utf-8')
      const { content, data } = readFrontmatter(fileContent)
      const title = (data && data.title) || ''
      const shortTitle = (data && data.shortTitle) || ''
      const intro = (data && data.intro) || ''
      for (const string of [content, title, shortTitle, intro]) {
        checkString(string, variables)
      }
    } catch (err) {
      if (err instanceof Error && 'code' in err && err.code === 'ENOENT') continue
      throw err
    }
  }
  for (const filePath of getReusableFiles()) {
    const fileContent = fs.readFileSync(filePath, 'utf-8')
    checkString(fileContent, variables)
  }

  const { outputFile, json } = options
  if (!outputFile || outputFile === 'stdout') {
    if (json) {
      console.log(JSON.stringify(Object.fromEntries(variables), null, 2))
    } else {
      console.log(variables)
    }
  } else if (options.markdown) {
    let output = ''
    const keys = Array.from(variables.values()).sort()
    if (keys.length > 0) {
      output += `There are ${variables.size} unused variables.\n\n`
      output += '| Variable | File |\n'
      output += '| --- | --- |\n'
      for (const key of keys) {
        output += `| ${key} | ${variables.get(key)} |\n`
      }
      output += `\nThis comment was generated by the \`find-unused-variables\` script.\n`
    }
    if (outputFile && output) {
      fs.writeFileSync(outputFile, output, 'utf-8')
    } else if (output) {
      console.log(output)
    }
  } else {
    if (json || outputFile.endsWith('.json')) {
      fs.writeFileSync(outputFile, JSON.stringify(Object.fromEntries(variables), null, 2), 'utf-8')
    } else {
      let output = ''
      for (const [key, value] of variables) {
        output += `${key} in ${value}\n`
      }
      fs.writeFileSync(outputFile, output, 'utf-8')
    }
  }
}

function getVariables(): Map<string, string> {
  const variables = new Map<string, string>()
  for (const filePath of walkFiles('data/variables', '.yml')) {
    const dottedPathBase = `variables.${filePath.replace('data/variables/', '').replace('.yml', '').replace(/\//g, '.')}`
    const data = yaml.load(fs.readFileSync(filePath, 'utf-8')) as Record<string, unknown>
    for (const key of Object.keys(data)) {
      const dottedPath = `${dottedPathBase}.${key}`
      variables.set(dottedPath, filePath)
    }
  }
  return variables
}

async function getPages() {
  const unversionedTree = await loadUnversionedTree([])
  const pageList = await loadPages(unversionedTree)
  return pageList
}

function getReusableFiles(root = 'data') {
  const here: string[] = []
  for (const file of fs.readdirSync(root)) {
    const filePath = `${root}/${file}`
    if (fs.statSync(filePath).isDirectory()) {
      here.push(...getReusableFiles(filePath))
    } else if (file.endsWith('.md') && file !== 'README.md') {
      here.push(filePath)
    }
  }
  return here
}

function checkString(string: string, variables: Map<string, string>) {
  try {
    const tokens = getLiquidTokens(string).filter(
      (token): token is TagToken => token.kind === TokenKind.Tag,
    )
    for (const token of tokens) {
      if (token.name === 'data') {
        const { args } = token
        variables.delete(args)
      }
    }
  } catch (err) {
    if (err instanceof TokenizationError) return
    throw err
  }
}