> ## Documentation Index
> Fetch the complete documentation index at: https://docs.endorlabs.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Instrumented container reachability

> <Badge color="green">Beta</Badge> <br /> Learn how to derive advanced OS package reachability when your container depends on complex external services.

export const YamlTable = ({children, data: propData, content}) => {
  const KV_RE = /^([A-Za-z][A-Za-z0-9_()/#\s-]+?):\s*(.+)$/;
  const INLINE_MD_RE = /(\[([^\]]+)\]\(([^)]+)\))|(`([^`]+)`)|(\*\*([^*]+)\*\*)|(\*([^*]+)\*)/g;
  const YES_RE = /^-yes-$/i;
  const NO_RE = /^-no-$/i;
  const LIMITED_RE = /^-(limited|partial)-$/i;
  const NA_RE = /^-(na|none)-$/i;
  const NA2_RE = /^-na2-$/i;
  const SIMPLE_TAG_RE = /(<br\s*\/?>)|(<p\s*\/?>)|(-note-)|(-warning-)/gi;
  const tryParseKV = trimmed => {
    const m = KV_RE.exec(trimmed);
    return m ? {
      key: m[1],
      value: m[2].trim()
    } : null;
  };
  const registerKey = (key, seenKeys, orderedKeys) => {
    if (!seenKeys.has(key)) {
      orderedKeys.push(key);
      seenKeys.add(key);
    }
  };
  const flushEntry = (currentEntry, entries) => {
    if (Object.keys(currentEntry).length > 0) entries.push(currentEntry);
  };
  const parseDashPrefixed = (lines, entries, orderedKeys, seenKeys) => {
    let currentEntry = {};
    let inEntry = false;
    for (const line of lines) {
      const trimmed = line.trim();
      if (trimmed.startsWith('- ')) {
        if (inEntry) entries.push(currentEntry);
        currentEntry = {};
        inEntry = true;
        const kv = tryParseKV(trimmed.substring(2).trim());
        if (kv) {
          registerKey(kv.key, seenKeys, orderedKeys);
          currentEntry[kv.key] = kv.value;
        }
      } else if (inEntry && trimmed !== '') {
        const kv = tryParseKV(trimmed);
        if (kv) {
          registerKey(kv.key, seenKeys, orderedKeys);
          currentEntry[kv.key] = kv.value;
        }
      }
    }
    flushEntry(currentEntry, entries);
  };
  const parseBlankSeparated = (lines, entries, orderedKeys, seenKeys) => {
    let currentEntry = {};
    let inEntry = false;
    for (const line of lines) {
      const trimmed = line.trim();
      if (trimmed === '') {
        if (inEntry) {
          flushEntry(currentEntry, entries);
          currentEntry = {};
          inEntry = false;
        }
        continue;
      }
      const kv = tryParseKV(trimmed);
      if (!kv) continue;
      const isNewEntry = !line.startsWith(' ') && !line.startsWith('\t');
      if (isNewEntry && inEntry && Object.keys(currentEntry).length > 0) {
        entries.push(currentEntry);
        currentEntry = {};
      }
      registerKey(kv.key, seenKeys, orderedKeys);
      currentEntry[kv.key] = kv.value;
      inEntry = true;
    }
    flushEntry(currentEntry, entries);
  };
  const normalizeEntries = (entries, orderedKeys) => entries.map(entry => {
    const filled = {};
    for (const key of orderedKeys) filled[key] = entry[key] || '';
    return filled;
  });
  const parseYamlTableContent = contentStr => {
    if (!contentStr) return [];
    const entries = [];
    const orderedKeys = [];
    const seenKeys = new Set();
    const lines = contentStr.split('\n');
    if (lines.some(line => line.trim().startsWith('- '))) {
      parseDashPrefixed(lines, entries, orderedKeys, seenKeys);
    } else {
      parseBlankSeparated(lines, entries, orderedKeys, seenKeys);
    }
    return normalizeEntries(entries, orderedKeys);
  };
  const processText = text => {
    if (!text) return text;
    const parts = [];
    let keyIndex = 0;
    let lastIndex = 0;
    let match;
    while ((match = INLINE_MD_RE.exec(text)) !== null) {
      if (match.index > lastIndex) parts.push(text.slice(lastIndex, match.index));
      if (match[1]) {
        parts.push(<a key={keyIndex++} href={match[3]}>{match[2]}</a>);
      } else if (match[4]) {
        parts.push(<code key={keyIndex++}>{match[5]}</code>);
      } else if (match[6]) {
        parts.push(<strong key={keyIndex++}>{match[7]}</strong>);
      } else if (match[8]) {
        parts.push(<em key={keyIndex++}>{match[9]}</em>);
      }
      lastIndex = match.index + match[0].length;
    }
    if (lastIndex < text.length) parts.push(text.slice(lastIndex));
    if (parts.length === 0) return text;
    const keyRef = {
      current: keyIndex
    };
    return expandHtmlTags(parts, keyRef);
  };
  const processBadges = text => {
    if (!text || typeof text !== 'string') return text;
    if (YES_RE.test(text)) return <span className="yt-badge-yes" role="img" aria-label="Supported" title="Supported">✓</span>;
    if (NO_RE.test(text)) return <span className="yt-badge-no" role="img" aria-label="Not supported" title="Not supported">✗</span>;
    if (LIMITED_RE.test(text)) return <span className="yt-badge-limited" role="img" aria-label="Partially supported" title="Partially supported">◐</span>;
    if (NA_RE.test(text) || NA2_RE.test(text)) return <span className="yt-sr-only" title="Not applicable">Not applicable</span>;
    return processText(text);
  };
  const cellClassName = text => {
    if (!text || typeof text !== 'string') return undefined;
    if (NA_RE.test(text)) return 'yt-cell-na';
    if (NA2_RE.test(text)) return 'yt-cell-na2';
    return undefined;
  };
  const expandSimpleTags = (str, keyRef) => {
    const result = [];
    let last = 0;
    SIMPLE_TAG_RE.lastIndex = 0;
    let m;
    while ((m = SIMPLE_TAG_RE.exec(str)) !== null) {
      if (m.index > last) result.push(str.slice(last, m.index));
      if (m[1]) {
        result.push(<br key={keyRef.current++} />);
      } else if (m[2]) {
        result.push(<br key={keyRef.current++} />, <br key={keyRef.current++} />);
      } else if (m[3]) {
        result.push(<span key={keyRef.current++} className="yt-badge-note" style={{
          fontWeight: 600
        }}>Note: </span>);
      } else if (m[4]) {
        result.push(<span key={keyRef.current++} className="yt-badge-warning" style={{
          fontWeight: 600
        }}>Warning: </span>);
      }
      last = m.index + m[0].length;
    }
    if (last < str.length) result.push(str.slice(last));
    return result;
  };
  const expandHtmlTags = (chunks, keyRef) => {
    const out = [];
    for (const chunk of chunks) {
      if (typeof chunk === 'string') {
        out.push(...expandSimpleTags(chunk, keyRef));
      } else {
        out.push(chunk);
      }
    }
    return out;
  };
  const extractText = node => {
    if (node === null || node === undefined) return '';
    if (typeof node === 'string') return node;
    if (typeof node === 'number') return String(node);
    if (typeof node === 'boolean') return '';
    if (Array.isArray(node)) return node.map(extractText).join('');
    if (node && typeof node === 'object' && node.type) {
      const props = node.props || ({});
      if (typeof props.children === 'string') return props.children;
      if (props.children) return extractText(props.children);
      return '';
    }
    return String(node || '');
  };
  const [mounted, setMounted] = useState(false);
  useEffect(() => {
    setMounted(true);
  }, []);
  const data = useMemo(() => {
    if (propData) return propData;
    if (content && typeof content === 'string') return parseYamlTableContent(content);
    if (!children) return [];
    if (typeof children === 'string') return parseYamlTableContent(children);
    const childrenArray = Array.isArray(children) ? children : [children];
    return parseYamlTableContent(childrenArray.map(extractText).join('').trim());
  }, [children, propData, content]);
  const columns = useMemo(() => {
    if (!data || data.length === 0) return [];
    const firstRow = data[0];
    if (!firstRow || typeof firstRow !== 'object') return [];
    return Object.keys(firstRow);
  }, [data]);
  if (!mounted) return null;
  if (!data || data.length === 0) return null;
  const rowKey = row => columns.map(c => row[c] || '').join('|');
  return <table>
      <thead>
        <tr>
          {columns.map(col => <th key={col}>{col.replaceAll('_', ' ')}</th>)}
        </tr>
      </thead>
      <tbody>
        {data.map(row => <tr key={rowKey(row)}>
            {columns.map(col => <td key={col} className={cellClassName(row[col])}>{processBadges(row[col])}</td>)}
          </tr>)}
      </tbody>
    </table>;
};

Instrumented container reachability is an advanced OS package reachability mode that embeds a runtime sensor in your container image to record how your application uses the image in a real environment. Use it when your container relies on complex external services that you cannot exercise during a short local run, so that reachability results then reflect actual usage.

Use instrumented reachability when:

* The container has complex external dependencies such as databases, message queues, third-party services that you cannot realistically exercise in a local ephemeral run.
* You want profiling to happen in a realistic environment, such as staging, that mirrors production traffic.
* You already have integration or end-to-end tests and want reachability to reflect those tests.

You can run instrumented container reachability using either [Kubernetes](#instrumented-reachability-kubernetes) or [Docker](#instrumented-reachability-docker).

<Note>
  Run the container scan using the new `endorctl container scan` command. The `endorctl scan --container` command does not support container reachability.
</Note>

### Prerequisites

To perform instrumented container reachability analysis, ensure that:

* Enable container scanning [using the `--os-reachability` flag](/scan/containers/container-reachability).
* Install and authenticate [endorctl](/developers-api/cli/install-and-configure).
* You have installed Docker daemon `dockerd` on the host and it runs and is accessible to the current user without elevated privileges. For example, `docker images` should work without `sudo`.
* The negotiated Docker API version between the client and server is `1.48` or higher.
* You must run the scan from either a Linux or a macOS host machine. Endor Labs supports container reachability on both amd64 and arm64 architectures.

To run the instrumented container images, you need either Docker or Kubernetes, based on your setup:

* **Docker**: Docker daemon and Docker CLI is available and you can run the instrumented image locally.
* **Kubernetes**: You have configured `kubectl` with access to your cluster so you can deploy and run the instrumented image in a pod.

### Determine instrumented container reachability using Kubernetes

Follow these steps to scan the original image, collect runtime profiling data, and determine reachability for containers using Kubernetes.

1. Run `endorctl container instrument` to create a new image with a lightweight sensor injected into the filesystem. The resulting image will have `-instrumented` appended to the tag.

   ```bash theme={null}
   endorctl container instrument \
     --image=<original_image>:<tag> \
     --app-stop-signal=QUIT \
     --load-instrumented-image=true
   ```

   * `--image`: Original container image to instrument.

   * `--app-stop-signal`: Signal used to stop the application. The sensor needs this so it can flush profiling data before the container exits.

   * `--load-instrumented-image`: Loads the instrumented image into your local Docker runtime so Kubernetes can reference it.

   * To instrument a multi-arch image for specific platforms, run `endorctl container instrument` with the `--platform` flag.

     ```bash theme={null}
     endorctl container instrument \
       --image=<image_name-tag> \
       --app-stop-signal=QUIT \
       --load-instrumented-image=true \
       --platform=linux/arm64,linux/amd64
     ```

2. Define how the instrumented image runs in a manifest. Create a manifest file such as `demo-manifest-file.yaml` to identify your workload. In the manifest, reference the instrumented image from step 1 and use a pod name and container name you can reuse later. You can also add `env`, `volumes`, and other options as needed for your application.

   ```yaml theme={null}
   apiVersion: v1
   kind: Pod
   metadata:
     name: <pod-name>
   spec:
     restartPolicy: OnFailure
     containers:
       - name: <container-name>
         image: <instrumented-image>
         ports:
           - containerPort: <container-port>
             hostPort: <host-port>
         securityContext:
           privileged: true
   ```

   * Set `securityContext.privileged` to `true` so the profiling sensor can run.
   * Set `restartPolicy` to `OnFailure` so that the pod does not restart automatically after you stop the app to generate the report.

3. Deploy the instrumented image to Kubernetes. The application runs normally while the sensor observes file access and process activity while the application runs.

   ```bash theme={null}
   kubectl apply -f <manifest-file>
   kubectl get pods <pod-name>
   ```

   Replace `<manifest-file>`, `<pod-name>`, and `<container-name>` with the values from your manifest.

   * If you need to access the application locally, run:

     ```bash theme={null}
     kubectl port-forward pod/<pod-name> <host-port>:<container-port>
     ```

   * You can also run your tests or interact with the application normally. The profiling sensor will capture runtime activity.

4. After you finish testing, send the `--app-stop-signal`, for example, `QUIT`, to stop the application gracefully. This signal triggers the profiling sensor to write the `creport.json` file and generate the profiling data.

   ```bash theme={null}
   kubectl exec -it <pod-name> -c <container-name> -- sh -c "kill -QUIT 1"
   ```

5. The sensor writes a profiling report to a known artifacts directory inside the container.

   * Verify that the `creport.json` file exists in the container:

     ```bash theme={null}
     kubectl exec -it <pod-name> -c <container-name> -- sh -c "ls -ls /opt/_instrumented/artifacts"
     ```

6. Create a local directory and copy the report to it.

   ```bash theme={null}
   # Create output directory
   mkdir -p collect_output
   # Copy the profiling data
   kubectl cp <pod-name>:/opt/_instrumented/artifacts/creport.json collect_output/creport.json -c <container-name>
   ```

   * To verify the copy, run:

     ```bash theme={null}
     ls -la collect_output/
     ```

   * Alternatively, you can use `endorctl container collect` to stop the running application and retrieve the profiling report from the instrumented container into a local directory. Skip steps 4, 5, and 6 if you are using this command.

     ```bash theme={null}
     endorctl container collect \
       --dynamic-profiling-data=true \
       --output-dir=collect_output \
       --image=<instrumented-image>
     ```

   * Set `--dynamic-profiling-data` to `true` to collect profiling data from the instrumented container.

   * Set `--output-dir` to the local directory that stores the collected data. The command creates a subdirectory under this path `cluster/pod/container`. Use that path for `--profiling-data-dir` in the next step.

7. Run `endorctl container scan` with the path to the directory that contains the collected profiling data, and OS reachability enabled. Endor Labs loads the report, maps runtime files to OS packages, and marks the corresponding packages as reachable.

   ```bash theme={null}
   endorctl container scan \
     --image=<original_image>:<tag> \
     --profiling-data-dir=collect_output \
     --project-name=<project-name> \
     --os-reachability
   ```

8. Remove the pod after completing the analysis:

   ```bash theme={null}
   kubectl delete pod <pod-name>
   ```

9. Optionally, publish the instrumented image to a registry using the `--publish` flag. The command pushes the image only if the Docker daemon is already authenticated with the target registry. `endorctl` will not attempt to re-authenticate with the container registry.

   ```bash theme={null}
   endorctl container instrument \
     --image=<image_name-tag> \
     --app-stop-signal=QUIT \
     --load-instrumented-image=true \
     --publish=true
   ```

### Determine instrumented container reachability using Docker

Follow these steps to scan the original image, collect runtime profiling data, and determine reachability for containers using Docker.

1. Run `endorctl container instrument` to create a new image with a lightweight sensor injected into the filesystem. At runtime, the sensor writes `creport.json` to `/opt/_instrumented/artifacts/`. By default, the command saves the image as `instrumented-image.tar`. To use a different output path, pass `--output-image-tar`.

   ```bash theme={null}
   endorctl container instrument \
     --image <original_image>:<tag> \
     --load-instrumented-image \
     --app-stop-signal <SIGNAL>
   ```

   * `--image`: Container image you want to instrument.
   * `--app-stop-signal`: Signal that stops the application so the sensor can flush profiling data before the container exits.
   * `--load-instrumented-image`: Loads the instrumented image into Docker so you can run it with `docker run`.

2. Run the instrumented image in Docker with `--privileged` so the sensor can use `ptrace` to watch file access. Docker blocks `ptrace` in unprivileged containers by default. You can run tests or interact with the application while the container runs. The sensor observes file access and process activity.

   ```bash theme={null}
   docker run -d \
     --name <container-name> \
     --privileged \
     <original_image>:<tag>-instrumented
   ```

   You can also pass:

   * `-e KEY=VALUE`, for example `-e PASSWORD=testpassword123`, to provide environment variables your app needs to start.
   * `-p <host-port>:<container-port>`, for example `-p 8080:8080`, to expose ports when you send traffic from your host during profiling.

3. After testing, send the `--app-stop-signal` to stop the application gracefully. This signal triggers the profiling sensor to write the `creport.json` file. The `sleep` gives the sensor time to finish writing.

   ```bash theme={null}
   docker kill --signal=<SIGNAL> <container-name>
   sleep 10
   ```

4. Create a local directory and copy the report to it.

   ```bash theme={null}
   mkdir -p ./profile-data
   docker cp <container-name>:/opt/_instrumented/artifacts/creport.json ./profile-data/
   ```

5. Run `endorctl container scan` with the path to the directory that contains the collected profiling data, and OS reachability enabled. Endor Labs loads the report, maps runtime files to OS packages, and marks the corresponding packages as reachable. If you run the command outside a Git repository, pass `--project-name <project-name>` to avoid an initialization error.

   ```bash theme={null}
   endorctl container scan \
     --image <original_image>:<tag> \
     --profiling-data-dir ./profile-data \
     --os-reachability \
     -n <your-namespace>
   ```

6. Remove the container after you complete the analysis.

   ```bash theme={null}
   docker rm <container-name>
   ```

### Instrumented reachability options

You can run the `endorctl container instrument` command with the following options.

<YamlTable>
  {`


    - Flag: \`app-stop-signal\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_APP_STOP_SIGNAL\`
    Type: string
    Description: Signal sent to the app so the sensor can flush profiling data before the container exits, for example, \`QUIT\` or \`TERM\`. Ensure the signal is compatible with your application.

    - Flag: \`app-stop-grace-period\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_APP_STOP_GRACE_PERIOD\`
    Type: string
    Description: Grace period for app shutdown, for example \`10s\`, \`1m\`. Use when the app needs time to flush before exit.

    - Flag: \`app-stderr-to-file\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_APP_STDERR_TO_FILE\`
    Type: boolean
    Description: Redirect application error output to a file in the instrumented container.

    - Flag: \`app-stdout-to-file\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_APP_STDOUT_TO_FILE\`
    Type: boolean
    Description: Redirect application standard output to a file in the instrumented container.

    - Flag: \`entrypoint\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_ENTRYPOINT\`
    Type: string
    Description: Override the image entrypoint (JSON array or shell string). Use when the image has a custom entrypoint.

    - Flag: \`cmd\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_CMD\`
    Type: string
    Description: Override the image CMD (JSON array or shell string). Use when the image has a custom CMD.

    - Flag: \`debug-mode\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_DEBUG_MODE\`
    Type: boolean
    Description: Enable sensor debug logs for instrumented container.

    - Flag: \`load-instrumented-image\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_LOAD_INSTRUMENTED_IMAGE\`
    Type: boolean
    Description: Load the instrumented image into the local Docker daemon so Kubernetes or a registry can use it. Default \`false\`.

    - Flag: \`output-image-tar\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_OUTPUT_IMAGE_TAR\`
    Type: string
    Description: Output tar path for the instrumented image. Default \`instrumented-image.tar\`.

    - Flag: \`sensor-path\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_SENSOR_PATH\`
    Type: string
    Description: Path to sensor binary.

    - Flag: \`platform\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_PLATFORM\`
    Type: string
    Description: Target platform for the instrumented image, for example \`linux/amd64\` or \`linux/arm64\`, or a comma-separated list such as \`linux/arm64,linux/amd64\` for multi-arch. Use when instrumenting multi-arch images. Requires \`--image\`.

    - Flag: \`publish\`
    Environment_Variable: \`ENDOR_CONTAINER_INSTRUMENT_PUBLISH\`
    Type: boolean
    Description: Publish the instrumented image to the registry after instrumentation completes. Requires you to authenticate the Docker daemon with the target registry. Default \`false\`.


    `}
</YamlTable>

You can run the `endorctl container collect` command with the following options.

<YamlTable>
  {`


    - Flag: \`dynamic-profiling-data\`
    Environment_Variable: \`ENDOR_CONTAINER_COLLECT_DYNAMIC_PROFILING_DATA\`
    Type: boolean
    Description: Collect dynamic profiling data from instrumented containers (default \`true\`).

    - Flag: \`kubeconfig-context\`
    Environment_Variable: \`ENDOR_CONTAINER_COLLECT_KUBECONFIG_CONTEXT\`
    Type: string
    Description: Provide the kubectl kubeconfig context to use to access the target (k8s) deployment environments.

    - Flag: \`kubeconfig-path\`
    Environment_Variable: \`ENDOR_CONTAINER_COLLECT_KUBECONFIG_PATH\`
    Type: string
    Description: Provide the kubectl kubeconfig path to use to access the target (k8s) deployment environments.

    - Flag: \`output-dir\`
    Environment_Variable: \`ENDOR_CONTAINER_COLLECT_OUTPUT_DIR\`
    Type: string
    Description: Set the directory to store collected data from the target deployment environment. The command creates a subdirectory \`cluster/pod/container\`. Use that path for \`--profiling-data-dir\` in the scan step.

    - Flag: \`runtime-type\`
    Environment_Variable: \`ENDOR_CONTAINER_COLLECT_RUNTIME_TYPE\`
    Type: string
    Description: Container runtime type. Supports \`k8s\` only.


    `}
</YamlTable>

### Troubleshoot issues

<AccordionGroup>
  <Accordion title="Why is profiling data not generated?">
    * Ensure that you send the QUIT signal correctly.

    * Check that the container has privileged: true in security context.

    * Verify that the `--app-stop-signal` matches the signal your application handles.
  </Accordion>

  <Accordion title="Why can't Kubernetes find my instrumented image?">
    * Run `kind load docker-image <image>` to load the image into kind.

    * Push the image to a container registry.
  </Accordion>

  <Accordion title="Why does my pod require privileged access?">
    The profiling sensor uses `ptrace` to watch file access, which unprivileged containers block by default. Set `securityContext.privileged: true` in the pod manifest.
  </Accordion>
</AccordionGroup>