> ## Documentation Index
> Fetch the complete documentation index at: https://docs.endorlabs.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Manage secret rules

> Use secret rules to scan and detect secrets

export const YamlTable = ({children, data: propData, content}) => {
  const KV_RE = /^([A-Za-z][A-Za-z0-9_()/#\s-]+?):\s*(.+)$/;
  const INLINE_MD_RE = /(\[([^\]]+)\]\(([^)]+)\))|(`([^`]+)`)|(\*\*([^*]+)\*\*)|(\*([^*]+)\*)/g;
  const YES_RE = /^-yes-$/i;
  const NO_RE = /^-no-$/i;
  const LIMITED_RE = /^-(limited|partial)-$/i;
  const NA_RE = /^-(na|none)-$/i;
  const NA2_RE = /^-na2-$/i;
  const SIMPLE_TAG_RE = /(<br\s*\/?>)|(<p\s*\/?>)|(-note-)|(-warning-)/gi;
  const tryParseKV = trimmed => {
    const m = KV_RE.exec(trimmed);
    return m ? {
      key: m[1],
      value: m[2].trim()
    } : null;
  };
  const registerKey = (key, seenKeys, orderedKeys) => {
    if (!seenKeys.has(key)) {
      orderedKeys.push(key);
      seenKeys.add(key);
    }
  };
  const flushEntry = (currentEntry, entries) => {
    if (Object.keys(currentEntry).length > 0) entries.push(currentEntry);
  };
  const parseDashPrefixed = (lines, entries, orderedKeys, seenKeys) => {
    let currentEntry = {};
    let inEntry = false;
    for (const line of lines) {
      const trimmed = line.trim();
      if (trimmed.startsWith('- ')) {
        if (inEntry) entries.push(currentEntry);
        currentEntry = {};
        inEntry = true;
        const kv = tryParseKV(trimmed.substring(2).trim());
        if (kv) {
          registerKey(kv.key, seenKeys, orderedKeys);
          currentEntry[kv.key] = kv.value;
        }
      } else if (inEntry && trimmed !== '') {
        const kv = tryParseKV(trimmed);
        if (kv) {
          registerKey(kv.key, seenKeys, orderedKeys);
          currentEntry[kv.key] = kv.value;
        }
      }
    }
    flushEntry(currentEntry, entries);
  };
  const parseBlankSeparated = (lines, entries, orderedKeys, seenKeys) => {
    let currentEntry = {};
    let inEntry = false;
    for (const line of lines) {
      const trimmed = line.trim();
      if (trimmed === '') {
        if (inEntry) {
          flushEntry(currentEntry, entries);
          currentEntry = {};
          inEntry = false;
        }
        continue;
      }
      const kv = tryParseKV(trimmed);
      if (!kv) continue;
      const isNewEntry = !line.startsWith(' ') && !line.startsWith('\t');
      if (isNewEntry && inEntry && Object.keys(currentEntry).length > 0) {
        entries.push(currentEntry);
        currentEntry = {};
      }
      registerKey(kv.key, seenKeys, orderedKeys);
      currentEntry[kv.key] = kv.value;
      inEntry = true;
    }
    flushEntry(currentEntry, entries);
  };
  const normalizeEntries = (entries, orderedKeys) => entries.map(entry => {
    const filled = {};
    for (const key of orderedKeys) filled[key] = entry[key] || '';
    return filled;
  });
  const parseYamlTableContent = contentStr => {
    if (!contentStr) return [];
    const entries = [];
    const orderedKeys = [];
    const seenKeys = new Set();
    const lines = contentStr.split('\n');
    if (lines.some(line => line.trim().startsWith('- '))) {
      parseDashPrefixed(lines, entries, orderedKeys, seenKeys);
    } else {
      parseBlankSeparated(lines, entries, orderedKeys, seenKeys);
    }
    return normalizeEntries(entries, orderedKeys);
  };
  const processText = text => {
    if (!text) return text;
    const parts = [];
    let keyIndex = 0;
    let lastIndex = 0;
    let match;
    while ((match = INLINE_MD_RE.exec(text)) !== null) {
      if (match.index > lastIndex) parts.push(text.slice(lastIndex, match.index));
      if (match[1]) {
        parts.push(<a key={keyIndex++} href={match[3]}>{match[2]}</a>);
      } else if (match[4]) {
        parts.push(<code key={keyIndex++}>{match[5]}</code>);
      } else if (match[6]) {
        parts.push(<strong key={keyIndex++}>{match[7]}</strong>);
      } else if (match[8]) {
        parts.push(<em key={keyIndex++}>{match[9]}</em>);
      }
      lastIndex = match.index + match[0].length;
    }
    if (lastIndex < text.length) parts.push(text.slice(lastIndex));
    if (parts.length === 0) return text;
    const keyRef = {
      current: keyIndex
    };
    return expandHtmlTags(parts, keyRef);
  };
  const processBadges = text => {
    if (!text || typeof text !== 'string') return text;
    if (YES_RE.test(text)) return <span className="yt-badge-yes" role="img" aria-label="Supported" title="Supported">✓</span>;
    if (NO_RE.test(text)) return <span className="yt-badge-no" role="img" aria-label="Not supported" title="Not supported">✗</span>;
    if (LIMITED_RE.test(text)) return <span className="yt-badge-limited" role="img" aria-label="Partially supported" title="Partially supported">◐</span>;
    if (NA_RE.test(text) || NA2_RE.test(text)) return <span className="yt-sr-only" title="Not applicable">Not applicable</span>;
    return processText(text);
  };
  const cellClassName = text => {
    if (!text || typeof text !== 'string') return undefined;
    if (NA_RE.test(text)) return 'yt-cell-na';
    if (NA2_RE.test(text)) return 'yt-cell-na2';
    return undefined;
  };
  const expandSimpleTags = (str, keyRef) => {
    const result = [];
    let last = 0;
    SIMPLE_TAG_RE.lastIndex = 0;
    let m;
    while ((m = SIMPLE_TAG_RE.exec(str)) !== null) {
      if (m.index > last) result.push(str.slice(last, m.index));
      if (m[1]) {
        result.push(<br key={keyRef.current++} />);
      } else if (m[2]) {
        result.push(<br key={keyRef.current++} />, <br key={keyRef.current++} />);
      } else if (m[3]) {
        result.push(<span key={keyRef.current++} className="yt-badge-note" style={{
          fontWeight: 600
        }}>Note: </span>);
      } else if (m[4]) {
        result.push(<span key={keyRef.current++} className="yt-badge-warning" style={{
          fontWeight: 600
        }}>Warning: </span>);
      }
      last = m.index + m[0].length;
    }
    if (last < str.length) result.push(str.slice(last));
    return result;
  };
  const expandHtmlTags = (chunks, keyRef) => {
    const out = [];
    for (const chunk of chunks) {
      if (typeof chunk === 'string') {
        out.push(...expandSimpleTags(chunk, keyRef));
      } else {
        out.push(chunk);
      }
    }
    return out;
  };
  const extractText = node => {
    if (node === null || node === undefined) return '';
    if (typeof node === 'string') return node;
    if (typeof node === 'number') return String(node);
    if (typeof node === 'boolean') return '';
    if (Array.isArray(node)) return node.map(extractText).join('');
    if (node && typeof node === 'object' && node.type) {
      const props = node.props || ({});
      if (typeof props.children === 'string') return props.children;
      if (props.children) return extractText(props.children);
      return '';
    }
    return String(node || '');
  };
  const [mounted, setMounted] = useState(false);
  useEffect(() => {
    setMounted(true);
  }, []);
  const data = useMemo(() => {
    if (propData) return propData;
    if (content && typeof content === 'string') return parseYamlTableContent(content);
    if (!children) return [];
    if (typeof children === 'string') return parseYamlTableContent(children);
    const childrenArray = Array.isArray(children) ? children : [children];
    return parseYamlTableContent(childrenArray.map(extractText).join('').trim());
  }, [children, propData, content]);
  const columns = useMemo(() => {
    if (!data || data.length === 0) return [];
    const firstRow = data[0];
    if (!firstRow || typeof firstRow !== 'object') return [];
    return Object.keys(firstRow);
  }, [data]);
  if (!mounted) return null;
  if (!data || data.length === 0) return null;
  const rowKey = row => columns.map(c => row[c] || '').join('|');
  return <table>
      <thead>
        <tr>
          {columns.map(col => <th key={col}>{col.replaceAll('_', ' ')}</th>)}
        </tr>
      </thead>
      <tbody>
        {data.map(row => <tr key={rowKey(row)}>
            {columns.map(col => <td key={col} className={cellClassName(row[col])}>{processBadges(row[col])}</td>)}
          </tr>)}
      </tbody>
    </table>;
};

You can use the following rules to scan your codebase and detect secrets:

* **System rules**: Endor Labs provides out-of-the-box rules for secret patterns for many public services like GitHub, GitLab, AWS, Bitbucket, Dropbox, and more.

* **Custom rules**: If you are using a service that is not included in the out-of-the-box list of secret patterns provided by Endor Labs, you can build your own custom rule to scan and detect the secrets for any service.

The following table lists the most important fields of the rule definition.

<YamlTable>
  {`
    - Field_name: \`meta.name\`
    Description: The display name of the rule.
    - Field_name: \`spec.rule_id\`
    Description: A unique identifier for the rule. Optional from the CLI and API. When omitted, Endor Labs generates one from the rule name. See [Automatic rule identifiers](#automatic-rule-identifiers).
    - Field_name: \`spec.description\`
    Description: A human-readable description of what the rule detects.
    - Field_name: \`spec.regex\`
    Description: The regular expression that the scanner matches against to detect a secret.
    - Field_name: \`spec.keywords\`
    Description: Strings used for a fast pre-filter before the regex is evaluated; a file is matched against the regex only when it contains a keyword. Optional, but recommended because it keeps scans fast.
    - Field_name: \`spec.path\`
    Description: A regular expression that limits the rule to matching file paths.
    - Field_name: \`spec.entropy\`
    Description: The minimum Shannon entropy a regex group must have to be considered a secret. Filters out low-randomness matches.
    - Field_name: \`spec.secret_group\`
    Description: The regex capture group whose value is extracted as the secret and checked against the entropy threshold.
    - Field_name: \`spec.tags\`
    Description: Labels attached to the rule for metadata and reporting.
    - Field_name: \`spec.validation\`
    Description: An HTTP request profile used to check whether a detected secret is active. See [Validator](#validator).
    - Field_name: \`spec.allowlists\`
    Description: One or more allowlists that exclude known false positives for this rule. See [Allowlists](#allowlists).
    - Field_name: \`spec.disabled\`
    Description: Set to \`true\` to exclude the rule from scans. Defaults to \`false\`.
    `}
</YamlTable>

### Automatic rule identifiers

When you create a secret rule from the CLI or API, `spec.rule_id` is optional. If you leave it empty, Endor Labs derives it from the rule name by lowercasing the name and replacing each run of non-alphanumeric characters with a hyphen. For example, a rule named `My Custom Rule` becomes `my-custom-rule`.

If the generated identifier already exists in your namespace, Endor Labs appends a short unique suffix, such as `my-custom-rule-3f1a9c2b`. A `rule_id` you supply yourself is always used unchanged.

<Note>
  Creating a secret rule in the Endor Labs user interface still requires a **Rule Identifier**. The automatic identifier applies to rules created with endorctl and the API.
</Note>

## Create a secret rule

1. Select **User menu** > **Policies & Rules** from the left sidebar.

2. Select **Secret Rules**.

3. Click **Create Secret Rules**.

   <img src="https://mintcdn.com/endorlabs-b4795f4f/Uc3T4mPoaFbRUPbf/images/scan/secrets/add-secret-rule.webp?fit=max&auto=format&n=Uc3T4mPoaFbRUPbf&q=85&s=bd64d97d350e77f7289d78ac1255b979" alt="Create secret rules" style={{width: '70%'}} width="1440" height="1256" data-path="images/scan/secrets/add-secret-rule.webp" />

4. Enter the unique **Rule Identifier** and **Rule Name**.

5. Enter the **Description** of the secret rule.

6. Enter the regex for the secret rule in **Detection Rule**.

7. Optionally, enter keywords for pre-regex filtering as comma-separated values in **Keywords**. Keywords are recommended because a file is matched against the regex only when it contains one of them, which keeps scans fast.

8. Optionally, enter the minimum Shannon entropy a regex group must have to be considered in **Entropy**.

   This helps filter out low randomness matches, like common words or predictable strings, and ensures only high-entropy values, more likely to be secrets, such as keys or tokens, are flagged.

9. Optionally, add validation details to validate the secret:

   * **Validation URL**: Enter the URL for validation.
   * **HTTP Method**: Choose `GET` or `POST`.
   * **Success Response Codes**: Enter valid response codes (For example, `200` for HTTP Status OK)
   * **Failure Response Codes**: Enter invalid response codes (For example, `401` for HTTP Status Unauthorized)
   * **Authorization Details**: Choose the authorization scheme used by the service: `Bearer`, `Token`, `Basic`, or `AWS4_HMAC_SHA256`.

10. Select **Propagate this rule to all child namespaces** to apply the secret rule to all child namespaces.

11. Click **Add Rule**.

### Create secret rules from the command line

For example, consider a token "demo\_value123" can be described using a regular expression. Here is an example of the rule specification:

```bash theme={null}
"meta": {
    "name": "Demo Token"
},
"spec": {
    "disabled": false,
    "keywords": [
        "demo_"
    ],
    "regex": "demo_[0-9a-zA-Z]{20}",
    "rule_id": "demo-rule"
}
```

Use the following command from the CLI to create this custom rule.

```bash expandable theme={null}
$ endorctl api create -r SecretRule -n demo  \
> --data '{
> "meta": {
>     "name": "Demo Token"
> },
> "spec": {
>     "disabled": false,
>     "keywords": [
>         "demo_"
>     ],
>     "regex": "demo_[0-9a-zA-Z]{20}",
>     "rule_id": "demo-rule"
> }
> }'
INFO: Initiating host-check ...
INFO: Host-check complete
{
  "meta": {
    "create_time": "2023-09-27T17:08:18.436936Z",
    "kind": "SecretRule",
    "name": "Demo Token",
    "update_time": "2023-09-27T17:08:18.436936Z",
    "upsert_time": "2023-09-27T17:08:18.436936Z",
    "version": "v1"
  },
  "spec": {
    "disabled": false,
    "keywords": [
      "demo_"
    ],
    "regex": "demo_[0-9a-zA-Z]{20}",
    "rule_id": "demo-rule"
  },
  "tenant_meta": {
    "namespace": "demo"
  },
  "uuid": "65146182aaeeffbaf5b6b553"
}
```

After the rule is created, Endor Labs uses this rule to detect this category of secrets.

To create a custom rule without a `rule_id`, omit the field and let Endor Labs generate one from the name.

```bash theme={null}
endorctl api create -r SecretRule -n <your-namespace> --data '{
  "meta": { "name": "Demo Token" },
  "spec": {
    "keywords": ["demo_"],
    "regex": "demo_[0-9a-zA-Z]{20}"
  }
}'
```

If you can validate the secret using an HTTP request, then you can also add [validation](#validator) to this rule. See the following example for creating a validation rule for a demo\_test123 token.

```bash theme={null}
curl -H "Authorization: Bearer "demo_test123" https://api.testserver.com/user
```

Then the validation specification can be:

```bash expandable theme={null}
"validation": {
    "name": "Demo secrets validator",
    "http_request": {
        "header": [
            {
                "key": "Bearer",
                "value": "{{.AuthzValue}}",
                "authz": true
            }
        ],
        "method": "GET",
        "uri": "https://api.testserver.com/user"
    },
    "http_response": {
        "failed_auth_codes": [
            401
        ],
        "successful_auth_codes": [
            200
        ]
    }
}
```

#### Validator

You can use a validator to check if a discovered secret is valid or not. The Endor Labs system rules for secrets include the necessary validator. When you validate a secret, the finding for that secret is categorized as critical, ensuring it receives higher priority compared to others.

When defining a custom rule, you can add your own validator from the command line or the Endor Labs user interface. Endor Labs uses this information to send an HTTP request, such as a GET or POST, to the address that the service specifies for the detected secret.

For example, when a GitHub Personal Access Token named "ghp\_endor123" is detected, Endor Labs sends the following HTTP request to GitHub's address:

```bash theme={null}
curl -H "Authorization: Token "ghp_endor123" https://api.github.com/user
```

The authentication codes defined by the service are used to mark the secrets as valid or invalid.

The validation portion of the secret rule contains the following fields:

<YamlTable>
  {`
    - Field: \`name\`
    Description: The name of the validator.
    - Field: \`description\`
    Description: A description of what the validator checks.
    - Field: \`http_request.uri\`
    Description: The address the validation request is sent to.
    - Field: \`http_request.method\`
    Description: The HTTP method to use, \`GET\` or \`POST\`.
    - Field: \`http_request.header\`
    Description: Key and value pairs added to the request header. Mark the entry that carries the secret with \`authz: true\`. See [HTTP request header](#http-request-header).
    - Field: \`http_response.successful_auth_codes\`
    Description: Response codes that mark the secret as valid, for example \`200\`.
    - Field: \`http_response.failed_auth_codes\`
    Description: Response codes that mark the secret as invalid, for example \`401\`.
    - Field: \`request_body\`
    Description: The request body to send, for validators that require a POST payload.
    - Field: \`hmac_auth\`
    Description: HMAC signing configuration, for services that require a signed request.
    - Field: \`template_params\`
    Description: Extra values Endor Labs supplies when the secret alone is not enough to validate, such as a tenant ID. Defined by Endor Labs for system rules. See [Template parameters](#template-parameters).
    `}
</YamlTable>

#### Template parameters

Some secrets cannot be validated with the secret value alone. For example, validating an Azure AD client secret also needs the client ID and tenant ID. The validator references these values in the request with template syntax, such as `{{.ClientID}}` and `{{.TenantID}}`, the same way `{{.AuthzValue}}` references the detected secret.

Endor Labs defines `template_params` for the system rules that need them. You can't set `template_params` on a custom rule from the CLI or API; Endor Labs manages these values.

A validated secret is marked valid or invalid based on the response codes. Endor Labs categorizes a valid secret as a critical finding so that it is prioritized for remediation.

#### HTTP request header

HTTP request header is a set of key-value pairs that should be added to the header.

```bash theme={null}
{
    "key": "Content-Type",
    "value": "application/json"
}
```

There are cases where one needs to use a value on runtime and substitute a pattern. For example, the secret itself that needs to be substituted is one such case. This is achieved by declaring a value using the `{{.Value}}` pattern.

For the HTTP header section that includes the secret, the block looks like the following snippet.

```bash theme={null}
{
    "key": "Token",
    "value": "{{.AuthzValue}}",
    "authz": true,
}
```

In this case, the scanner replaces the candidate secret that was detected and adds it to the HTTP request header in place of `{{.AuthzValue}}`.

The following table describes a special case where the key-value pair is marked with the `authz` flag and is used to craft the "Authorization" part of the header, where three options are supported.

<YamlTable>
  {`


    - Key: Basic
    Header: 'Authorization: \`Basic "hash{{.AuthzValue}}"’\`'
    - Key: Bearer
    Header: 'Authorization: \`Bearer {{.AuthzValue}}\`'
    - Key: Token
    Header: 'Authorization: \`Token {{.AuthzValue}}\`'


    `}
</YamlTable>

## Allowlists

An allowlist excludes known false positives for a rule without disabling the rule. Add one or more allowlists under `spec.allowlists`. Each allowlist supports the following fields.

<YamlTable>
  {`
    - Field: \`description\`
    Description: A short description of what the allowlist excludes.
    - Field: \`regexes\`
    Description: Content patterns to ignore.
    - Field: \`regex_target\`
    Description: What the \`regexes\` match against. Use \`match\` for the regex match, \`secret\` for the captured secret, or \`line\` for the whole line.
    - Field: \`paths\`
    Description: File path patterns to ignore.
    - Field: \`commits\`
    Description: Commit SHAs to ignore.
    - Field: \`stop_words\`
    Description: Strings that, when present in the secret, mark the match as a false positive.
    - Field: \`condition\`
    Description: How the criteria combine. \`OR\` ignores a match when any criterion matches. \`AND\` ignores a match only when all criteria match. Defaults to \`OR\`.
    `}
</YamlTable>

The following example ignores test fixtures and any match that contains `EXAMPLE`.

```json theme={null}
"allowlists": [
  {
    "description": "Ignore test fixtures and example values",
    "paths": ["(^|/)testdata/"],
    "stop_words": ["EXAMPLE"],
    "condition": "OR"
  }
]
```

## Manage secret rules

1. Select **User menu** > **Policies & Rules** from the left sidebar.

2. Select **Secret Rules**.

   The list of all secret rules appears.

   <img src="https://mintcdn.com/endorlabs-b4795f4f/Uc3T4mPoaFbRUPbf/images/scan/secrets/secret-rules.webp?fit=max&auto=format&n=Uc3T4mPoaFbRUPbf&q=85&s=e20b9ffb8fdbc59b95d7dcb6ae4dc8de" alt="Secret rules" style={{width: '80%'}} width="1648" height="926" data-path="images/scan/secrets/secret-rules.webp" />

3. Select the rule for which you want to view the details.

   The rule details appear in the right sidebar.

   <img src="https://mintcdn.com/endorlabs-b4795f4f/Uc3T4mPoaFbRUPbf/images/scan/secrets/secret-rule-details.webp?fit=max&auto=format&n=Uc3T4mPoaFbRUPbf&q=85&s=fc4e38b8387e2de52189df033238fabd" alt="Secret rule details" style={{width: '50%'}} width="936" height="1292" data-path="images/scan/secrets/secret-rule-details.webp" />

### Clone a secret rule

Select the vertical three dots to the right of the rule, then select **Clone Rule**.

The cloned rule appears in the list of secret rules and you can edit it.

### Edit a secret rule

Select the vertical three dots to the right of the rule, then select **Edit Rule**.

You can only edit the custom rules that you created or the system rules that you cloned.

### Fetch secret rules with endorctl

To fetch the Endor Labs secret scanning rules from the command line type the following commands:

```bash theme={null}
endorctl api list -r SecretRule -n <your-namespace>
```

For example, to see the rule for the GitHub Personal Access Token, you could search by the name `GitHub Personal Access Token` or by the rule-id `github-pat`:

```bash theme={null}
endorctl api get -r SecretRule -n <your-namespace> --name "GitHub Personal Access Token"
endorctl api list -r SecretRule -n <your-namespace> --filter=spec.rule_id==github-pat
```

To export all rules, including system rules, as YAML for offline scans, use the `--list-all` flag with YAML output.

```bash theme={null}
endorctl api list -r SecretRule -o yaml --list-all > secret-rules.yaml
```